run IAM purge routines deterministically every hr (#20587)

Existing implementation runs IAM purge routines for expired LDAP and
OIDC accounts with a probability of 0.25 after every IAM refresh. This
change ensures that they are run once in each hour.

.github/workflows/go-lint.yml

@@ -21,23 +21,13 @@ jobs:
     strategy:
       matrix:
         go-version: [1.22.x]
-        os: [ubuntu-latest, Windows]
+        os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - name: Build on ${{ matrix.os }}
-        if: matrix.os == 'Windows'
-        env:
-          CGO_ENABLED: 0
-          GO111MODULE: on
-        run: |
-          Set-MpPreference -DisableRealtimeMonitoring $true
-          netsh int ipv4 set dynamicport tcp start=60000 num=61000
-          go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
-          go test -v --timeout 120m ./...
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/iam-integrations.yaml

@@ -125,3 +125,37 @@ jobs:
         if: matrix.openid == 'http://127.0.0.1:5556/dex'
         run: |
           make test-site-replication-oidc
+  iam-import-with-missing-entities:
+    name: Test IAM import in new cluster with missing entities
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Checkout minio-iam-testing
+        uses: actions/checkout@v4
+        with:
+          repository: minio/minio-iam-testing
+          path: minio-iam-testing
+      - name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
+        run: |
+          make test-iam-import-with-missing-entities
+  iam-import-with-openid:
+    name: Test IAM import in new cluster with opendid configurations
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Checkout minio-iam-testing
+        uses: actions/checkout@v4
+        with:
+          repository: minio/minio-iam-testing
+          path: minio-iam-testing
+      - name: Test import of IAM artifacts when in fresh cluster with openid configurations
+        run: |
+          make test-iam-import-with-openid

.github/workflows/replication.yaml

@@ -70,3 +70,9 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-versioning
+
+      - name: Test Multipart upload with failures
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-multipart

.github/workflows/vulncheck.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.5
+          go-version: 1.22.7
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
         shell: bash

CONTRIBUTING.md

@@ -12,8 +12,9 @@ Fork [MinIO upstream](https://github.com/minio/minio/fork) source repository to
 
 ```sh
 git clone https://github.com/minio/minio
+cd minio
 go install -v
-ls /go/bin/minio
+ls $(go env GOPATH)/bin/minio
 ```
 
 ### Set up git remote as ``upstream``

Dockerfile

@@ -1,5 +1,7 @@
 FROM minio/minio:latest
 
+RUN chmod -R 777 /usr/bin
+
 COPY ./minio /usr/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 

Dockerfile.dev

@@ -1,12 +0,0 @@
-FROM minio/minio:latest
-
-ENV PATH=/opt/bin:$PATH
-
-COPY ./minio /opt/bin/minio
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.hotfix

@@ -1,24 +1,26 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine as build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
@@ -51,9 +53,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

Dockerfile.release

@@ -1,35 +1,39 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
+
+WORKDIR /build
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
+    apk add -U --no-cache bash && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
-RUN if [ "$TARGETARCH" = "amd64" ]; then \
-       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
-       chmod +x /go/bin/curl; \
-    fi
-
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
 
+COPY dockerscripts/download-static-curl.sh /build/download-static-curl
+RUN chmod +x /build/download-static-curl && \
+    /build/download-static-curl
+
 FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
 ARG RELEASE
@@ -51,10 +55,12 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/mc* /usr/bin/
-COPY --from=build /go/bin/cur* /usr/bin/
+COPY --from=build /go/bin/curl* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Dockerfile.release.fips

@@ -1,21 +1,28 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
+# Download mc binary and signature files
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.sha256sum -o /go/bin/mc.sha256sum && \
+    chmod +x /go/bin/mc
+
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
        curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
        chmod +x /go/bin/curl; \
@@ -44,8 +51,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
     MINIO_CONFIG_ENV_FILE=config.env
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

Dockerfile.release.old_cpu

@@ -1,24 +1,26 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
+ENV GOPATH=/go
-ENV CGO_ENABLED 0
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
@@ -51,9 +53,11 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
+RUN chmod -R 777 /usr/bin
+
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

Makefile

@@ -24,7 +24,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.2.3-0.20241022140105-4558fbf3a223
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -97,6 +97,14 @@ test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP)
 	@echo "Running upgrade tests for IAM (LDAP backend)"
 	@env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh
 
+test-iam-import-with-missing-entities: install-race ## test import of external iam config withg missing entities
+	@echo "Test IAM import configurations with missing entities"
+	@env bash $(PWD)/docs/distributed/iam-import-with-missing-entities.sh
+
+test-iam-import-with-openid: install-race
+	@echo "Test IAM import configurations with openid"
+	@env bash $(PWD)/docs/distributed/iam-import-with-openid.sh
+
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
 
@@ -133,6 +141,10 @@ test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site"
 	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh)
 
+test-multipart: install-race ## test multipart
+	@echo "Test multipart behavior when part files are missing"
+	@(env bash $(PWD)/buildscripts/multipart-quorum-test.sh)
+
 verify: install-race ## verify minio various setups
 	@echo "Verifying build with race"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)

README.md

@@ -93,7 +93,6 @@ The following table lists supported architectures. Replace the `wget` URL with t
 | 64-bit Intel/AMD               | <https://dl.min.io/server/minio/release/linux-amd64/minio>   |
 | 64-bit ARM                     | <https://dl.min.io/server/minio/release/linux-arm64/minio>   |
 | 64-bit PowerPC LE (ppc64le)    | <https://dl.min.io/server/minio/release/linux-ppc64le/minio> |
-| IBM Z-Series (S390X)           | <https://dl.min.io/server/minio/release/linux-s390x/minio>   |
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 

buildscripts/multipart-quorum-test.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+	set -x
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+trap 'catch $LINENO' ERR
+
+function purge() {
+	rm -rf "$1"
+}
+
+# shellcheck disable=SC2120
+catch() {
+	if [ $# -ne 0 ]; then
+		echo "error on line $1"
+	fi
+
+	echo "Cleaning up instances of MinIO"
+	pkill minio || true
+	pkill -9 minio || true
+	purge "$WORK_DIR"
+	if [ $# -ne 0 ]; then
+		exit $#
+	fi
+}
+
+catch
+
+function start_minio_10drive() {
+	start_port=$1
+
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	export MINIO_CI_CD=1
+
+	mkdir ${WORK_DIR}
+	C_PWD=${PWD}
+	if [ ! -x "$PWD/mc" ]; then
+		MC_BUILD_DIR="mc-$RANDOM"
+		if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+			echo "failed to download https://github.com/minio/mc"
+			purge "${MC_BUILD_DIR}"
+			exit 1
+		fi
+
+		(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+		# remove mc source.
+		purge "${MC_BUILD_DIR}"
+	fi
+
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...10}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 5
+
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	"${PWD}/mc" mb --with-versioning minio/bucket
+
+	export AWS_ACCESS_KEY_ID=minio
+	export AWS_SECRET_ACCESS_KEY=minio123
+	aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
+	uploadId=$(jq -r '.UploadId' upload-id.json)
+
+	truncate -s 5MiB file-5mib
+	for i in {1..2}; do
+		aws --endpoint-url http://localhost:"$start_port" s3api upload-part \
+			--upload-id "$uploadId" --bucket bucket --key obj-1 \
+			--part-number "$i" --body ./file-5mib
+	done
+	for i in {1..6}; do
+		find ${WORK_DIR}/disk${i}/.minio.sys/multipart/ -type f -name "part.1" -delete
+	done
+	cat <<EOF >parts.json
+{
+    "Parts": [
+        {
+            "PartNumber": 1,
+            "ETag": "5f363e0e58a95f06cbe9bbc662c5dfb6"
+        },
+        {
+            "PartNumber": 2,
+            "ETag": "5f363e0e58a95f06cbe9bbc662c5dfb6"
+        }
+    ]
+}
+EOF
+	err=$(aws --endpoint-url http://localhost:"$start_port" s3api complete-multipart-upload --upload-id "$uploadId" --bucket bucket --key obj-1 --multipart-upload file://./parts.json 2>&1)
+	rv=$?
+	if [ $rv -eq 0 ]; then
+		echo "Failed to receive an error"
+		exit 1
+	fi
+	echo "Received an error during complete-multipart as expected: $err"
+}
+
+function main() {
+	start_port=$(shuf -i 10000-65000 -n 1)
+	start_minio_10drive ${start_port}
+}
+
+main "$@"

cmd/admin-bucket-handlers.go

@@ -427,6 +427,21 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			cfgPath := pathJoin(bi.Name, cfgFile)
 			bucket := bi.Name
 			switch cfgFile {
+			case bucketPolicyConfig:
+				config, _, err := globalBucketMetadataSys.GetBucketPolicy(bucket)
+				if err != nil {
+					if errors.Is(err, BucketPolicyNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := json.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketNotificationConfig:
 				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
 				if err != nil {
@@ -796,11 +811,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 
 			bucketMap[bucket].NotificationConfigXML = configData
+			bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
 			if sz > maxBucketPolicySize {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
+				rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyTooLarge.String()))
 				continue
 			}
 
@@ -818,7 +834,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Version in policy must not be empty
 			if bucketPolicy.Version == "" {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyInvalidVersion.String()))
+				rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyInvalidVersion.String()))
 				continue
 			}
 

cmd/admin-handlers-idp-ldap.go

@@ -190,7 +190,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 //
 // PUT /minio/admin/v3/idp/ldap/add-service-account
 func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
-	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, true)
 	if APIError.Code != "" {
 		writeErrorResponseJSON(ctx, w, APIError, r.URL)
 		return
@@ -499,7 +499,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 
 	dnList := r.Form["userDNs"]
 	isAll := r.Form.Get("all") == "true"
-	onlySelf := !isAll && len(dnList) == 0
+	selfOnly := !isAll && len(dnList) == 0
 
 	if isAll && len(dnList) > 0 {
 		// This should be checked on client side, so return generic error
@@ -527,7 +527,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 			dn = foundResult.NormDN
 		}
 		if dn == cred.ParentUser || dnList[0] == cred.ParentUser {
-			onlySelf = true
+			selfOnly = true
 		}
 	}
 
@@ -538,13 +538,13 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-		DenyOnly:        onlySelf,
+		DenyOnly:        selfOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
 	}
 
-	if onlySelf && len(dnList) == 0 {
+	if selfOnly && len(dnList) == 0 {
 		selfDN := cred.AccessKey
 		if cred.ParentUser != "" {
 			selfDN = cred.ParentUser
@@ -609,10 +609,9 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 				return
 			}
 			for _, sts := range stsKeys {
-				expiryTime := sts.Expiration
 				accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
 					AccessKey:  sts.AccessKey,
-					Expiration: &expiryTime,
+					Expiration: &sts.Expiration,
 				})
 			}
 			// if only STS keys, skip if user has no STS keys
@@ -628,10 +627,9 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 				return
 			}
 			for _, svc := range serviceAccounts {
-				expiryTime := svc.Expiration
 				accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
 					AccessKey:  svc.AccessKey,
-					Expiration: &expiryTime,
+					Expiration: &svc.Expiration,
 				})
 			}
 			// if only service accounts, skip if user has no service accounts

cmd/admin-handlers-users.go

@@ -26,15 +26,16 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"slices"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 	"unicode/utf8"
 
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/mux"
 	xldap "github.com/minio/pkg/v3/ldap"
@@ -636,7 +637,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, false)
 	if APIError.Code != "" {
 		writeErrorResponseJSON(ctx, w, APIError, r.URL)
 		return
@@ -1166,6 +1167,172 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
+// ListAccessKeysBulk - GET /minio/admin/v3/list-access-keys-bulk
+func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	users := r.Form["users"]
+	isAll := r.Form.Get("all") == "true"
+	selfOnly := !isAll && len(users) == 0
+
+	if isAll && len(users) > 0 {
+		// This should be checked on client side, so return generic error
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	// Empty user list and not self, list access keys for all users
+	if isAll {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListUsersAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else if len(users) == 1 {
+		if users[0] == cred.AccessKey || users[0] == cred.ParentUser {
+			selfOnly = true
+		}
+	}
+
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.ListServiceAccountsAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        selfOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if selfOnly && len(users) == 0 {
+		selfUser := cred.AccessKey
+		if cred.ParentUser != "" {
+			selfUser = cred.ParentUser
+		}
+		users = append(users, selfUser)
+	}
+
+	var checkedUserList []string
+	if isAll {
+		users, err := globalIAMSys.ListUsers(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		for user := range users {
+			checkedUserList = append(checkedUserList, user)
+		}
+		checkedUserList = append(checkedUserList, globalActiveCred.AccessKey)
+	} else {
+		for _, user := range users {
+			// Validate the user
+			_, ok := globalIAMSys.GetUser(ctx, user)
+			if !ok {
+				continue
+			}
+			checkedUserList = append(checkedUserList, user)
+		}
+	}
+
+	listType := r.Form.Get("listType")
+	var listSTSKeys, listServiceAccounts bool
+	switch listType {
+	case madmin.AccessKeyListUsersOnly:
+		listSTSKeys = false
+		listServiceAccounts = false
+	case madmin.AccessKeyListSTSOnly:
+		listSTSKeys = true
+		listServiceAccounts = false
+	case madmin.AccessKeyListSvcaccOnly:
+		listSTSKeys = false
+		listServiceAccounts = true
+	case madmin.AccessKeyListAll:
+		listSTSKeys = true
+		listServiceAccounts = true
+	default:
+		err := errors.New("invalid list type")
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+
+	accessKeyMap := make(map[string]madmin.ListAccessKeysResp)
+	for _, user := range checkedUserList {
+		accessKeys := madmin.ListAccessKeysResp{}
+		if listSTSKeys {
+			stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, user)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, sts := range stsKeys {
+				accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
+					AccessKey:  sts.AccessKey,
+					Expiration: &sts.Expiration,
+				})
+			}
+			// if only STS keys, skip if user has no STS keys
+			if !listServiceAccounts && len(stsKeys) == 0 {
+				continue
+			}
+		}
+
+		if listServiceAccounts {
+			serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, user)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, svc := range serviceAccounts {
+				accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
+					AccessKey:  svc.AccessKey,
+					Expiration: &svc.Expiration,
+				})
+			}
+			// if only service accounts, skip if user has no service accounts
+			if !listSTSKeys && len(serviceAccounts) == 0 {
+				continue
+			}
+		}
+		accessKeyMap[user] = accessKeys
+	}
+
+	data, err := json.Marshal(accessKeyMap)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}
+
 // AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -1235,18 +1402,6 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	bucketStorageCache.InitOnce(10*time.Second,
-		cachevalue.Opts{ReturnLastGood: true},
-		func(ctx context.Context) (DataUsageInfo, error) {
-			ctx, done := context.WithTimeout(ctx, 2*time.Second)
-			defer done()
-
-			return loadDataUsageFromBackend(ctx, objectAPI)
-		},
-	)
-
-	dataUsageInfo, _ := bucketStorageCache.Get()
-
 	// If etcd, dns federation configured list buckets from etcd.
 	var err error
 	var buckets []BucketInfo
@@ -1287,7 +1442,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 
 	var buf []byte
 	switch {
-	case accountName == globalActiveCred.AccessKey:
+	case accountName == globalActiveCred.AccessKey || newGlobalAuthZPluginFn() != nil:
+		// For owner account and when plugin authZ is configured always set
+		// effective policy as `consoleAdmin`.
+		//
+		// In the latter case, we let the UI render everything, but individual
+		// actions would fail if not permitted by the external authZ service.
 		for _, policy := range policy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
 				effectivePolicy = policy.Definition
@@ -1333,15 +1493,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		rd, wr := isAllowedAccess(bucket.Name)
 		if rd || wr {
 			// Fetch the data usage of the current bucket
-			var size uint64
+			bui := globalBucketQuotaSys.GetBucketUsageInfo(ctx, bucket.Name)
-			var objectsCount uint64
+			size := bui.Size
-			var objectsHist, versionsHist map[string]uint64
+			objectsCount := bui.ObjectsCount
-			if !dataUsageInfo.LastUpdate.IsZero() {
+			objectsHist := bui.ObjectSizesHistogram
-				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
+			versionsHist := bui.ObjectVersionsHistogram
-				objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
+
-				objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
-				versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
-			}
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
 			if enablePrefixUsage {
@@ -2062,6 +2219,16 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 
 // ImportIAM - imports all IAM info into MinIO
 func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
+	a.importIAM(w, r, "")
+}
+
+// ImportIAMV2 - imports all IAM info into MinIO
+func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) {
+	a.importIAM(w, r, "v2")
+}
+
+// ImportIAM - imports all IAM info into MinIO
+func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) {
 	ctx := r.Context()
 
 	// Get current object layer instance.
@@ -2086,6 +2253,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+
+	var skipped, removed, added madmin.IAMEntities
+	var failed madmin.IAMErrEntities
+
 	// import policies first
 	{
 
@@ -2111,8 +2282,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			for policyName, policy := range allPolicies {
 				if policy.IsEmpty() {
 					err = globalIAMSys.DeletePolicy(ctx, policyName, true)
+					removed.Policies = append(removed.Policies, policyName)
 				} else {
 					_, err = globalIAMSys.SetPolicy(ctx, policyName, policy)
+					added.Policies = append(added.Policies, policyName)
 				}
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allPoliciesFile, policyName), r.URL)
@@ -2191,8 +2364,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					return
 				}
 				if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
-					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, toAdminAPIErrCode(ctx, err), err, allUsersFile, accessKey), r.URL)
+					failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err})
-					return
+				} else {
+					added.Users = append(added.Users, accessKey)
 				}
 
 			}
@@ -2230,8 +2404,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					}
 				}
 				if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, gerr, allGroupsFile, group), r.URL)
+					failed.Groups = append(failed.Groups, madmin.IAMErrEntity{Name: group, Error: err})
-					return
+				} else {
+					added.Groups = append(added.Groups, group)
 				}
 			}
 		}
@@ -2260,7 +2435,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 
 			// Validations for LDAP enabled deployments.
 			if globalIAMSys.LDAPConfig.Enabled() {
-				err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs)
+				skippedAccessKeys, err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs)
+				skipped.ServiceAccounts = append(skipped.ServiceAccounts, skippedAccessKeys...)
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, ""), r.URL)
 					return
@@ -2268,6 +2444,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 
 			for user, svcAcctReq := range serviceAcctReqs {
+				if slices.Contains(skipped.ServiceAccounts, user) {
+					continue
+				}
 				var sp *policy.Policy
 				var err error
 				if len(svcAcctReq.SessionPolicy) > 0 {
@@ -2325,10 +2504,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				}
 
 				if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+					failed.ServiceAccounts = append(failed.ServiceAccounts, madmin.IAMErrEntity{Name: user, Error: err})
-					return
+				} else {
+					added.ServiceAccounts = append(added.ServiceAccounts, user)
 				}
-
 			}
 		}
 	}
@@ -2365,8 +2544,15 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					return
 				}
 				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
+					failed.UserPolicies = append(
-					return
+						failed.UserPolicies,
+						madmin.IAMErrPolicyEntity{
+							Name:     u,
+							Policies: strings.Split(pm.Policies, ","),
+							Error:    err,
+						})
+				} else {
+					added.UserPolicies = append(added.UserPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")})
 				}
 			}
 		}
@@ -2396,7 +2582,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			// Validations for LDAP enabled deployments.
 			if globalIAMSys.LDAPConfig.Enabled() {
 				isGroup := true
-				err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap)
+				skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap)
+				skipped.Groups = append(skipped.Groups, skippedDN...)
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, ""), r.URL)
 					return
@@ -2404,9 +2591,19 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 
 			for g, pm := range grpPolicyMap {
+				if slices.Contains(skipped.Groups, g) {
+					continue
+				}
 				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
+					failed.GroupPolicies = append(
-					return
+						failed.GroupPolicies,
+						madmin.IAMErrPolicyEntity{
+							Name:     g,
+							Policies: strings.Split(pm.Policies, ","),
+							Error:    err,
+						})
+				} else {
+					added.GroupPolicies = append(added.GroupPolicies, map[string][]string{g: strings.Split(pm.Policies, ",")})
 				}
 			}
 		}
@@ -2436,13 +2633,17 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			// Validations for LDAP enabled deployments.
 			if globalIAMSys.LDAPConfig.Enabled() {
 				isGroup := true
-				err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap)
+				skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap)
+				skipped.Users = append(skipped.Users, skippedDN...)
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, ""), r.URL)
 					return
 				}
 			}
 			for u, pm := range userPolicyMap {
+				if slices.Contains(skipped.Users, u) {
+					continue
+				}
 				// disallow setting policy mapping if user is a temporary user
 				ok, _, err := globalIAMSys.IsTempUser(u)
 				if err != nil && err != errNoSuchUser {
@@ -2455,11 +2656,35 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				}
 
 				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
+					failed.STSPolicies = append(
+						failed.STSPolicies,
+						madmin.IAMErrPolicyEntity{
+							Name:     u,
+							Policies: strings.Split(pm.Policies, ","),
+							Error:    err,
+						})
+				} else {
+					added.STSPolicies = append(added.STSPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")})
+				}
+			}
+		}
+	}
+
+	if apiVer == "v2" {
+		iamr := madmin.ImportIAMResult{
+			Skipped: skipped,
+			Removed: removed,
+			Added:   added,
+			Failed:  failed,
+		}
+
+		b, err := json.Marshal(iamr)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-			}
+
-		}
+		writeSuccessResponseJSON(w, b)
 	}
 }
 
@@ -2475,7 +2700,7 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
 	return nil
 }
 
-func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
+func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
 	ctx := r.Context()
 
 	// Get current object layer instance.
@@ -2542,6 +2767,14 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
 	}
 
+	denyOnly := (targetUser == cred.AccessKey || targetUser == cred.ParentUser)
+	if ldap && !denyOnly {
+		res, _ := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser)
+		if res.NormDN == cred.ParentUser {
+			denyOnly = true
+		}
+	}
+
 	// Check if action is allowed if creating access key for another user
 	// Check if action is explicitly denied if for self
 	if !globalIAMSys.IsAllowed(policy.Args{
@@ -2551,7 +2784,7 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 		ConditionValues: condValues,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-		DenyOnly:        (targetUser == cred.AccessKey || targetUser == cred.ParentUser),
+		DenyOnly:        denyOnly,
 	}) {
 		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
 	}

cmd/admin-handlers-users_test.go

@@ -28,6 +28,7 @@ import (
 	"net/http"
 	"net/url"
 	"runtime"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -337,16 +338,24 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -442,7 +451,7 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
     "s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::testbucket/*"
+    "arn:aws:s3:::testbucket"
    ]
   }
  ]
@@ -559,16 +568,24 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -669,16 +686,24 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 
 	// Check that default policies can be overwritten.
 	err = s.adm.AddCannedPolicy(ctx, "readwrite", policyBytes)
@@ -714,16 +739,24 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -770,8 +803,9 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	if err != nil {
 		c.Fatalf("group list err: %v", err)
 	}
-	if !set.CreateStringSet(groups...).Contains(group) {
+	expected := []string{group}
-		c.Fatalf("created group not present!")
+	if !slices.Equal(groups, expected) {
+		c.Fatalf("expected group listing: %v, got: %v", expected, groups)
 	}
 	groupInfo, err := s.adm.GetGroupDescription(ctx, group)
 	if err != nil {
@@ -877,16 +911,24 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -961,16 +1003,24 @@ func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -1043,16 +1093,24 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -1600,7 +1658,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
     "s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::%s/*"
+    "arn:aws:s3:::%s"
    ]
   }
  ]

cmd/admin-handlers.go

@@ -93,11 +93,18 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
+	if globalInplaceUpdateDisabled {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
 	}
 
+	if currentReleaseTime.IsZero() || currentReleaseTime.Equal(timeSentinel) {
+		apiErr := errorCodes.ToAPIErr(ErrMethodNotAllowed)
+		apiErr.Description = fmt.Sprintf("unable to perform in-place update, release time is unrecognized: %s", currentReleaseTime)
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
+		return
+	}
+
 	vars := mux.Vars(r)
 	updateURL := vars["updateURL"]
 	dryRun := r.Form.Get("dry-run") == "true"
@@ -110,6 +117,11 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		}
 	}
 
+	local := globalLocalNodeName
+	if local == "" {
+		local = "127.0.0.1"
+	}
+
 	u, err := url.Parse(updateURL)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -128,6 +140,39 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		return
 	}
 
+	updateStatus := madmin.ServerUpdateStatusV2{
+		DryRun:  dryRun,
+		Results: make([]madmin.ServerPeerUpdateStatus, 0, len(globalNotificationSys.allPeerClients)),
+	}
+	peerResults := make(map[string]madmin.ServerPeerUpdateStatus, len(globalNotificationSys.allPeerClients))
+	failedClients := make(map[int]bool, len(globalNotificationSys.allPeerClients))
+
+	if lrTime.Sub(currentReleaseTime) <= 0 {
+		updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
+			Host:           local,
+			Err:            fmt.Sprintf("server is running the latest version: %s", Version),
+			CurrentVersion: Version,
+		})
+
+		for _, client := range globalNotificationSys.peerClients {
+			updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
+				Host:           client.String(),
+				Err:            fmt.Sprintf("server is running the latest version: %s", Version),
+				CurrentVersion: Version,
+			})
+		}
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(updateStatus)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		writeSuccessResponseJSON(w, jsonBytes)
+		return
+	}
+
 	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
 	// Download Binary Once
 	binC, bin, err := downloadBinary(u, mode)
@@ -137,16 +182,6 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	updateStatus := madmin.ServerUpdateStatusV2{DryRun: dryRun}
-	peerResults := make(map[string]madmin.ServerPeerUpdateStatus)
-
-	local := globalLocalNodeName
-	if local == "" {
-		local = "127.0.0.1"
-	}
-
-	failedClients := make(map[int]struct{})
-
 	if globalIsDistErasure {
 		// Push binary to other servers
 		for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
@@ -156,7 +191,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 					Err:            nerr.Err.Error(),
 					CurrentVersion: Version,
 				}
-				failedClients[idx] = struct{}{}
+				failedClients[idx] = true
 			} else {
 				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
 					Host:           nerr.Host.String(),
@@ -167,7 +202,6 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		}
 	}
 
-	if lrTime.Sub(currentReleaseTime) > 0 {
 	if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
 		peerResults[local] = madmin.ServerPeerUpdateStatus{
 			Host:           local,
@@ -181,20 +215,12 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 			UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 		}
 	}
-	} else {
-		peerResults[local] = madmin.ServerPeerUpdateStatus{
-			Host:           local,
-			Err:            fmt.Sprintf("server is already running the latest version: %s", Version),
-			CurrentVersion: Version,
-		}
-	}
 
 	if !dryRun {
 		if globalIsDistErasure {
 			ng := WithNPeers(len(globalNotificationSys.peerClients))
 			for idx, client := range globalNotificationSys.peerClients {
-				_, ok := failedClients[idx]
+				if failedClients[idx] {
-				if ok {
 					continue
 				}
 				client := client
@@ -240,14 +266,14 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		startTime := time.Now().Add(restartUpdateDelay)
 		ng := WithNPeers(len(globalNotificationSys.peerClients))
 		for idx, client := range globalNotificationSys.peerClients {
-			_, ok := failedClients[idx]
+			if failedClients[idx] {
-			if ok {
 				continue
 			}
 			client := client
 			ng.Go(ctx, func() error {
 				prs, ok := peerResults[client.String()]
-				if ok && prs.CurrentVersion != prs.UpdatedVersion && prs.UpdatedVersion != "" {
+				// We restart only on success, not for any failures.
+				if ok && prs.Err == "" {
 					return client.SignalService(serviceRestart, "", dryRun, &startTime)
 				}
 				return nil
@@ -284,7 +310,9 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 	writeSuccessResponseJSON(w, jsonBytes)
 
 	if !dryRun {
-		if lrTime.Sub(currentReleaseTime) > 0 {
+		prs, ok := peerResults[local]
+		// We restart only on success, not for any failures.
+		if ok && prs.Err == "" {
 			globalServiceSignalCh <- serviceRestart
 		}
 	}
@@ -1036,7 +1064,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	// Start profiling on remote servers.
 	var hostErrs []NotificationPeerErr
 	for _, profiler := range profiles {
-		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(profiler)...)
+		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(ctx, profiler)...)
 
 		// Start profiling locally as well.
 		prof, err := startProfiler(profiler)
@@ -1117,7 +1145,11 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 
 	// Start profiling on remote servers.
 	for _, profiler := range profiles {
-		globalNotificationSys.StartProfiling(profiler)
+		// Limit start time to max 10s.
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		globalNotificationSys.StartProfiling(ctx, profiler)
+		// StartProfiling blocks, so we can cancel now.
+		cancel()
 
 		// Start profiling locally as well.
 		prof, err := startProfiler(profiler)
@@ -1132,6 +1164,10 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	for {
 		select {
 		case <-ctx.Done():
+			// Stop remote profiles
+			go globalNotificationSys.DownloadProfilingData(GlobalContext, io.Discard)
+
+			// Stop local
 			globalProfilerMu.Lock()
 			defer globalProfilerMu.Unlock()
 			for k, v := range globalProfiler {
@@ -2331,6 +2367,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 }
 
 func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) madmin.InfoMessage {
+	const operationTimeout = 10 * time.Second
 	ldap := madmin.LDAP{}
 	if globalIAMSys.LDAPConfig.Enabled() {
 		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
@@ -2371,7 +2408,9 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 		mode = madmin.ItemOnline
 
 		// Load data usage
-		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+		ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
+		dataUsageInfo, err := loadDataUsageFromBackend(ctx2, objectAPI)
+		cancel()
 		if err == nil {
 			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
 			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
@@ -2405,18 +2444,24 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 		}
 
 		if pools {
-			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
+			ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
+			poolsInfo, _ = getPoolsInfo(ctx2, allDisks)
+			cancel()
 		}
 	}
 
 	domain := globalDomainNames
 	services := madmin.Services{
-		KMSStatus:     fetchKMSStatus(ctx),
 		LDAP:          ldap,
 		Logger:        log,
 		Audit:         audit,
 		Notifications: notifyTarget,
 	}
+	{
+		ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
+		services.KMSStatus = fetchKMSStatus(ctx2)
+		cancel()
+	}
 
 	return madmin.InfoMessage{
 		Mode:          string(mode),
@@ -3050,7 +3095,7 @@ func targetStatus(ctx context.Context, h logger.Target) madmin.Status {
 	return madmin.Status{Status: string(madmin.ItemOffline)}
 }
 
-// fetchLoggerDetails return log info
+// fetchLoggerInfo return log info
 func fetchLoggerInfo(ctx context.Context) ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit

cmd/admin-heal-ops.go

@@ -806,18 +806,20 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
 }
 
-func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+func (h *healSequence) healItems(objAPI ObjectLayer) error {
 	if h.clientToken == bgHealingUUID {
 		// For background heal do nothing.
 		return nil
 	}
 
+	if h.bucket == "" { // heal internal meta only during a site-wide heal
 		if err := h.healDiskMeta(objAPI); err != nil {
 			return err
 		}
+	}
 
 	// Heal buckets and objects
-	return h.healBuckets(objAPI, bucketsOnly)
+	return h.healBuckets(objAPI)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -828,8 +830,7 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
-	bucketsOnly := false // Heals buckets and objects also.
+	h.traverseAndHealDoneCh <- h.healItems(objAPI)
-	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
 	xioutil.SafeClose(h.traverseAndHealDoneCh)
 }
 
@@ -856,14 +857,14 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
+func (h *healSequence) healBuckets(objAPI ObjectLayer) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(objAPI, h.bucket, bucketsOnly)
+		return h.healBucket(objAPI, h.bucket, false)
 	}
 
 	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
@@ -877,7 +878,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	})
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
+		if err = h.healBucket(objAPI, bucket.Name, false); err != nil {
 			return err
 		}
 	}

cmd/admin-router.go

@@ -159,14 +159,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// Info operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(adminMiddleware(adminAPI.ServerInfoHandler, traceAllFlag, noObjLayerFlag))
-		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceAllFlag))
+		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceHdrsS3HFlag))
 
 		// StorageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
 		// Metrics operation
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceAllFlag))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceHdrsS3HFlag))
 
 		if globalIsDistErasure || globalIsErasure {
 			// Heal operations
@@ -193,9 +193,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// Profiling operations - deprecated API
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(adminMiddleware(adminAPI.StartProfilingHandler, traceAllFlag, noObjLayerFlag)).
 			Queries("profilerType", "{profilerType:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceAllFlag, noObjLayerFlag))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceHdrsS3HFlag, noObjLayerFlag))
 		// Profiling operations
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceAllFlag, noObjLayerFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceHdrsS3HFlag, noObjLayerFlag))
 
 		// Config KV operations.
 		if enableConfigOps {
@@ -244,6 +244,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// STS accounts ops
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}")
 
+		// Access key (service account/STS) operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-access-keys-bulk").HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysBulk)).Queries("listType", "{listType:.*}")
+
 		// Info policy IAM latest
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 		// List policies latest
@@ -290,6 +293,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// Import IAM info
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam-v2").HandlerFunc(adminMiddleware(adminAPI.ImportIAMV2, noGZFlag))
 
 		// IDentity Provider configuration APIs
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg))

cmd/api-errors.go

@@ -28,7 +28,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/minio/minio/internal/ioutil"
 	"google.golang.org/api/googleapi"
 
@@ -978,7 +978,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationNoExistingObjects: {
 		Code:           "XMinioReplicationNoExistingObjects",
-		Description:    "No matching ExistingsObjects rule enabled",
+		Description:    "No matching ExistingObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrRemoteTargetDenyAddError: {
@@ -1486,7 +1486,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrTooManyRequests: {
 		Code:           "TooManyRequests",
-		Description:    "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
+		Description:    "Please reduce your request rate",
 		HTTPStatusCode: http.StatusTooManyRequests,
 	},
 	ErrUnsupportedMetadata: {
@@ -2549,11 +2549,11 @@ func toAPIError(ctx context.Context, err error) APIError {
 			if len(e.Errors) >= 1 {
 				apiErr.Code = e.Errors[0].Reason
 			}
-		case azblob.StorageError:
+		case *azcore.ResponseError:
 			apiErr = APIError{
-				Code:           string(e.ServiceCode()),
+				Code:           e.ErrorCode,
 				Description:    e.Error(),
-				HTTPStatusCode: e.Response().StatusCode,
+				HTTPStatusCode: e.StatusCode,
 			}
 			// Add more other SDK related errors here if any in future.
 		default:
@@ -2592,7 +2592,7 @@ func getAPIError(code APIErrorCode) APIError {
 	return errorCodes.ToAPIErr(ErrInternalError)
 }
 
-// getErrorResponse gets in standard error and resource value and
+// getAPIErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
 func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID, hostID string) APIErrorResponse {
 	reqInfo := logger.GetReqInfo(ctx)

cmd/api-response.go

@@ -593,8 +593,9 @@ func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, v
 			for k, v := range cleanReservedKeys(object.UserDefined) {
 				content.UserMetadata.Set(k, v)
 			}
-
+			if !object.Expires.IsZero() {
 				content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
+			}
 			content.Internal = &ObjectInternalInfo{
 				K: object.DataBlocks,
 				M: object.ParityBlocks,
@@ -729,7 +730,9 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
 				for k, v := range cleanReservedKeys(object.UserDefined) {
 					content.UserMetadata.Set(k, v)
 				}
+				if !object.Expires.IsZero() {
 					content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
+				}
 				content.Internal = &ObjectInternalInfo{
 					K: object.DataBlocks,
 					M: object.ParityBlocks,
@@ -947,19 +950,10 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 // writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	switch err.HTTPStatusCode {
-	case http.StatusServiceUnavailable:
+	case http.StatusServiceUnavailable, http.StatusTooManyRequests:
 		// Set retry-after header to indicate user-agents to retry request after 60 seconds.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "60")
-	case http.StatusTooManyRequests:
-		_, deadline := globalAPIConfig.getRequestsPool()
-		if deadline <= 0 {
-			// Set retry-after header to indicate user-agents to retry request after 10 seconds.
-			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-			w.Header().Set(xhttp.RetryAfter, "10")
-		} else {
-			w.Header().Set(xhttp.RetryAfter, strconv.Itoa(int(deadline.Seconds())))
-		}
 	}
 
 	switch err.Code {

cmd/api-router.go

@@ -456,6 +456,14 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
 			Queries("cors", "")
+		// PutBucketCors - this is a dummy call.
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
+			Queries("cors", "")
+		// DeleteBucketCors - this is a dummy call.
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
+			Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)).
@@ -472,6 +480,7 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)).
 			Queries("logging", "")
+
 		// GetBucketTaggingHandler
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)).

cmd/auth-handler.go

@@ -222,7 +222,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }
 
-func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
+func getClaimsFromTokenWithSecret(token, secret string) (*xjwt.MapClaims, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -244,7 +244,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 
 	// If AuthZPlugin is set, return without any further checks.
 	if newGlobalAuthZPluginFn() != nil {
-		return claims.Map(), nil
+		return claims, nil
 	}
 
 	// Check if a session policy is set. If so, decode it here.
@@ -263,12 +263,16 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
 	}
 
-	return claims.Map(), nil
+	return claims, nil
 }
 
 // Fetch claims in the security token returned by the client.
 func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+	jwtClaims, err := getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+	return jwtClaims.Map(), nil
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
@@ -319,7 +323,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		if err != nil {
 			return nil, toAPIErrorCode(r.Context(), err)
 		}
-		return claims, ErrNone
+		return claims.Map(), ErrNone
 	}
 
 	claims := xjwt.NewMapClaims()

cmd/background-newdisks-heal-ops.go

@@ -74,8 +74,10 @@ type healingTracker struct {
 	// for resuming with correct numbers.
 	ResumeItemsHealed  uint64 `json:"-"`
 	ResumeItemsFailed  uint64 `json:"-"`
+	ResumeItemsSkipped uint64 `json:"-"`
 	ResumeBytesDone    uint64 `json:"-"`
 	ResumeBytesFailed  uint64 `json:"-"`
+	ResumeBytesSkipped uint64 `json:"-"`
 
 	// Filled on startup/restarts.
 	QueuedBuckets []string
@@ -90,6 +92,9 @@ type healingTracker struct {
 	BytesSkipped uint64
 
 	RetryAttempts uint64
+
+	Finished bool // finished healing, whether with errors or not
+
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -143,6 +148,26 @@ func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	return h
 }
 
+func (h *healingTracker) resetHealing() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.ItemsHealed = 0
+	h.ItemsFailed = 0
+	h.BytesDone = 0
+	h.BytesFailed = 0
+	h.ResumeItemsHealed = 0
+	h.ResumeItemsFailed = 0
+	h.ResumeBytesDone = 0
+	h.ResumeBytesFailed = 0
+	h.ItemsSkipped = 0
+	h.BytesSkipped = 0
+
+	h.HealedBuckets = nil
+	h.Object = ""
+	h.Bucket = ""
+}
+
 func (h *healingTracker) getLastUpdate() time.Time {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
@@ -198,9 +223,6 @@ func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) {
 // update will update the tracker on the disk.
 // If the tracker has been deleted an error is returned.
 func (h *healingTracker) update(ctx context.Context) error {
-	if h.disk.Healing() == nil {
-		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
-	}
 	h.mu.Lock()
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
@@ -262,8 +284,10 @@ func (h *healingTracker) resume() {
 
 	h.ItemsHealed = h.ResumeItemsHealed
 	h.ItemsFailed = h.ResumeItemsFailed
+	h.ItemsSkipped = h.ResumeItemsSkipped
 	h.BytesDone = h.ResumeBytesDone
 	h.BytesFailed = h.ResumeBytesFailed
+	h.BytesSkipped = h.ResumeBytesSkipped
 }
 
 // bucketDone should be called when a bucket is done healing.
@@ -274,8 +298,10 @@ func (h *healingTracker) bucketDone(bucket string) {
 
 	h.ResumeItemsHealed = h.ItemsHealed
 	h.ResumeItemsFailed = h.ItemsFailed
+	h.ResumeItemsSkipped = h.ItemsSkipped
 	h.ResumeBytesDone = h.BytesDone
 	h.ResumeBytesFailed = h.BytesFailed
+	h.ResumeBytesSkipped = h.BytesSkipped
 	h.HealedBuckets = append(h.HealedBuckets, bucket)
 	for i, b := range h.QueuedBuckets {
 		if b == bucket {
@@ -324,6 +350,7 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		PoolIndex:         h.PoolIndex,
 		SetIndex:          h.SetIndex,
 		DiskIndex:         h.DiskIndex,
+		Finished:          h.Finished,
 		Path:              h.Path,
 		Started:           h.Started.UTC(),
 		LastUpdate:        h.LastUpdate.UTC(),
@@ -339,6 +366,7 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		Object:            h.Object,
 		QueuedBuckets:     h.QueuedBuckets,
 		HealedBuckets:     h.HealedBuckets,
+		RetryAttempts:     h.RetryAttempts,
 
 		ObjectsHealed: h.ItemsHealed, // Deprecated July 2021
 		ObjectsFailed: h.ItemsFailed, // Deprecated July 2021
@@ -353,11 +381,13 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	}
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
-
+	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn {
-	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
+
+	go globalMRFState.startMRFPersistence()
+	go globalMRFState.healRoutine(z)
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
@@ -370,7 +400,7 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 			continue
 		}
-		if disk.Healing() != nil {
+		if h := disk.Healing(); h != nil && !h.Finished {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 		}
 	}
@@ -469,16 +499,19 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	// if objects have failed healing, we attempt a retry to heal the drive upto 3 times before giving up.
 	if tracker.ItemsFailed > 0 && tracker.RetryAttempts < 4 {
 		tracker.RetryAttempts++
-		bugLogIf(ctx, tracker.update(ctx))
 
 		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retrying %s time (healed: %d, skipped: %d, failed: %d).", disk,
 			humanize.Ordinal(int(tracker.RetryAttempts)), tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+
+		tracker.resetHealing()
+		bugLogIf(ctx, tracker.update(ctx))
+
 		return errRetryHealing
 	}
 
 	if tracker.ItemsFailed > 0 {
 		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retried %d times (healed: %d, skipped: %d, failed: %d).", disk,
-			tracker.RetryAttempts-1, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+			tracker.RetryAttempts, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 	} else {
 		if tracker.RetryAttempts > 0 {
 			healingLogEvent(ctx, "Healing of drive '%s' is complete, retried %d times (healed: %d, skipped: %d).", disk,
@@ -516,7 +549,8 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 			continue
 		}
 		if t.HealID == tracker.HealID {
-			t.delete(ctx)
+			t.Finished = true
+			t.update(ctx)
 		}
 	}
 

cmd/background-newdisks-heal-ops_gen.go

@@ -132,6 +132,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ResumeItemsFailed")
 				return
 			}
+		case "ResumeItemsSkipped":
+			z.ResumeItemsSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeItemsSkipped")
+				return
+			}
 		case "ResumeBytesDone":
 			z.ResumeBytesDone, err = dc.ReadUint64()
 			if err != nil {
@@ -144,6 +150,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ResumeBytesFailed")
 				return
 			}
+		case "ResumeBytesSkipped":
+			z.ResumeBytesSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesSkipped")
+				return
+			}
 		case "QueuedBuckets":
 			var zb0002 uint32
 			zb0002, err = dc.ReadArrayHeader()
@@ -206,6 +218,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "RetryAttempts")
 				return
 			}
+		case "Finished":
+			z.Finished, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "Finished")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -219,9 +237,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 26
+	// map header, size 29
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x1a, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -400,6 +418,16 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ResumeItemsFailed")
 		return
 	}
+	// write "ResumeItemsSkipped"
+	err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeItemsSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeItemsSkipped")
+		return
+	}
 	// write "ResumeBytesDone"
 	err = en.Append(0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
 	if err != nil {
@@ -420,6 +448,16 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ResumeBytesFailed")
 		return
 	}
+	// write "ResumeBytesSkipped"
+	err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeBytesSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeBytesSkipped")
+		return
+	}
 	// write "QueuedBuckets"
 	err = en.Append(0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
 	if err != nil {
@@ -494,15 +532,25 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "RetryAttempts")
 		return
 	}
+	// write "Finished"
+	err = en.Append(0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.Finished)
+	if err != nil {
+		err = msgp.WrapError(err, "Finished")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 26
+	// map header, size 29
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x1a, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -555,12 +603,18 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ResumeItemsFailed"
 	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.ResumeItemsFailed)
+	// string "ResumeItemsSkipped"
+	o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeItemsSkipped)
 	// string "ResumeBytesDone"
 	o = append(o, 0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
 	o = msgp.AppendUint64(o, z.ResumeBytesDone)
 	// string "ResumeBytesFailed"
 	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.ResumeBytesFailed)
+	// string "ResumeBytesSkipped"
+	o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeBytesSkipped)
 	// string "QueuedBuckets"
 	o = append(o, 0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.QueuedBuckets)))
@@ -585,6 +639,9 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "RetryAttempts"
 	o = append(o, 0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
 	o = msgp.AppendUint64(o, z.RetryAttempts)
+	// string "Finished"
+	o = append(o, 0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.Finished)
 	return
 }
 
@@ -714,6 +771,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ResumeItemsFailed")
 				return
 			}
+		case "ResumeItemsSkipped":
+			z.ResumeItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeItemsSkipped")
+				return
+			}
 		case "ResumeBytesDone":
 			z.ResumeBytesDone, bts, err = msgp.ReadUint64Bytes(bts)
 			if err != nil {
@@ -726,6 +789,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ResumeBytesFailed")
 				return
 			}
+		case "ResumeBytesSkipped":
+			z.ResumeBytesSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesSkipped")
+				return
+			}
 		case "QueuedBuckets":
 			var zb0002 uint32
 			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -788,6 +857,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "RetryAttempts")
 				return
 			}
+		case "Finished":
+			z.Finished, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Finished")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -802,7 +877,7 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *healingTracker) Msgsize() (s int) {
-	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
+	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
 	for za0001 := range z.QueuedBuckets {
 		s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001])
 	}
@@ -810,6 +885,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
-	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size
+	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size + 9 + msgp.BoolSize
 	return
 }

cmd/batch-expire.go

@@ -27,7 +27,6 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -342,8 +341,24 @@ func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versio
 		PrefixEnabledFn:  vc.PrefixEnabled,
 		VersionSuspended: vc.Suspended(),
 	}
-	_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel, opts)
+
-	return errs
+	allErrs := make([]error, 0, len(objsToDel))
+
+	for {
+		count := len(objsToDel)
+		if count == 0 {
+			break
+		}
+		if count > maxDeleteList {
+			count = maxDeleteList
+		}
+		_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
+		allErrs = append(allErrs, errs...)
+		// Next batch of deletion
+		objsToDel = objsToDel[count:]
+	}
+
+	return allErrs
 }
 
 const (
@@ -373,9 +388,12 @@ func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
 
 func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
 	vc, _ := globalBucketVersioningSys.Get(r.Bucket)
-	retryAttempts := r.Retry.Attempts
+	retryAttempts := job.Expire.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchExpireJobDefaultRetries
+	}
 	delay := job.Expire.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchExpireJobDefaultRetryDelay
 	}
 
@@ -541,9 +559,13 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	results := make(chan itemOrErr[ObjectInfo], workerSize)
 	go func() {
-		for _, prefix := range r.Prefix.F() {
+		prefixes := r.Prefix.F()
+		if len(prefixes) == 0 {
+			prefixes = []string{""}
+		}
+		for _, prefix := range prefixes {
 			prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
-			err := api.Walk(ctx, r.Bucket, strings.TrimSpace(prefix), prefixResultCh, WalkOptions{
+			err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
 				Marker:       lastObject,
 				LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
 				VersionsSort: WalkVersionsSortDesc,

cmd/batch-handlers.go

@@ -276,8 +276,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	globalBatchJobsMetrics.save(job.ID, ri)
 
+	retryAttempts := job.Replicate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchReplJobDefaultRetries
+	}
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchReplJobDefaultRetryDelay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
@@ -297,12 +301,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return true
 		}
 
-		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(oi.ModTime) {
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(oi.ModTime) {
 			// skip all objects that are created before the specified time.
 			return true
 		}
 
-		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(oi.ModTime) {
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(oi.ModTime) {
 			// skip all objects that are created after the specified time.
 			return true
 		}
@@ -377,7 +381,6 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		return err
 	}
 
-	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
@@ -388,9 +391,13 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 
 		objInfoCh := make(chan miniogo.ObjectInfo, 1)
 		go func() {
-			for _, prefix := range r.Source.Prefix.F() {
+			prefixes := r.Source.Prefix.F()
+			if len(prefixes) == 0 {
+				prefixes = []string{""}
+			}
+			for _, prefix := range prefixes {
 				prefixObjInfoCh := c.ListObjects(ctx, r.Source.Bucket, miniogo.ListObjectsOptions{
-					Prefix:       strings.TrimSpace(prefix),
+					Prefix:       prefix,
 					WithVersions: minioSrc,
 					Recursive:    true,
 					WithMetadata: true,
@@ -768,22 +775,10 @@ func (ri *batchJobInfo) loadOrInit(ctx context.Context, api ObjectLayer, job Bat
 		switch {
 		case job.Replicate != nil:
 			ri.Version = batchReplVersionV1
-			ri.RetryAttempts = batchReplJobDefaultRetries
-			if job.Replicate.Flags.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
-			}
 		case job.KeyRotate != nil:
 			ri.Version = batchKeyRotateVersionV1
-			ri.RetryAttempts = batchKeyRotateJobDefaultRetries
-			if job.KeyRotate.Flags.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
-			}
 		case job.Expire != nil:
 			ri.Version = batchExpireVersionV1
-			ri.RetryAttempts = batchExpireJobDefaultRetries
-			if job.Expire.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.Expire.Retry.Attempts
-			}
 		}
 		return nil
 	}
@@ -1030,10 +1025,15 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
+	retryAttempts := job.Replicate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchReplJobDefaultRetries
+	}
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay < time.Second {
+	if delay <= 0 {
 		delay = batchReplJobDefaultRetryDelay
 	}
+
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	selectObj := func(info FileInfo) (ok bool) {
@@ -1047,12 +1047,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			return false
 		}
 
-		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(info.ModTime) {
 			// skip all objects that are created before the specified time.
 			return false
 		}
 
-		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(info.ModTime) {
 			// skip all objects that are created after the specified time.
 			return false
 		}
@@ -1121,7 +1121,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
@@ -1204,9 +1203,13 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
 		go func() {
-			for _, prefix := range r.Source.Prefix.F() {
+			prefixes := r.Source.Prefix.F()
+			if len(prefixes) == 0 {
+				prefixes = []string{""}
+			}
+			for _, prefix := range prefixes {
 				prefixWalkCh := make(chan itemOrErr[ObjectInfo], 100)
-				if err := api.Walk(ctx, r.Source.Bucket, strings.TrimSpace(prefix), prefixWalkCh, WalkOptions{
+				if err := api.Walk(ctx, r.Source.Bucket, prefix, prefixWalkCh, WalkOptions{
 					Marker:   lastObject,
 					Filter:   selectObj,
 					AskDisks: walkQuorum,
@@ -1574,9 +1577,6 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 	}
 
 	jobType := r.Form.Get("jobType")
-	if jobType == "" {
-		jobType = string(madmin.BatchJobReplicate)
-	}
 
 	resultCh := make(chan itemOrErr[ObjectInfo])
 
@@ -1605,7 +1605,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 			continue
 		}
 
-		if jobType == string(req.Type()) {
+		if jobType == string(req.Type()) || jobType == "" {
 			listResult.Jobs = append(listResult.Jobs, madmin.BatchJobResult{
 				ID:      req.ID,
 				Type:    req.Type(),
@@ -1705,7 +1705,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 	w.Write(buf)
 }
 
-// StarBatchJob queue a new job for execution
+// StartBatchJob queue a new job for execution
 func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -1986,7 +1986,6 @@ func (j *BatchJobPool) AddWorker() {
 					}
 				}
 			}
-			job.delete(j.ctx, j.objLayer)
 			j.canceler(job.ID, false)
 		case <-j.workerKillCh:
 			return
@@ -2047,7 +2046,9 @@ func (j *BatchJobPool) canceler(jobID string, cancel bool) error {
 			canceler()
 		}
 	}
+	if cancel {
 		delete(j.jobCancelers, jobID)
+	}
 	return nil
 }
 
@@ -2173,11 +2174,42 @@ func (m *batchJobMetrics) purgeJobMetrics() {
 			m.RUnlock()
 			for _, jobID := range toDeleteJobMetrics {
 				m.delete(jobID)
+				j := BatchJobRequest{
+					ID: jobID,
+				}
+				j.delete(GlobalContext, newObjectLayerFn())
 			}
 		}
 	}
 }
 
+// load metrics from disk on startup
+func (m *batchJobMetrics) init(ctx context.Context, objectAPI ObjectLayer) error {
+	resultCh := make(chan itemOrErr[ObjectInfo])
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, resultCh, WalkOptions{}); err != nil {
+		return err
+	}
+
+	for result := range resultCh {
+		if result.Err != nil {
+			return result.Err
+		}
+		ri := &batchJobInfo{}
+		if err := ri.loadByPath(ctx, objectAPI, result.Item.Name); err != nil {
+			if !errors.Is(err, errNoSuchJob) {
+				batchLogIf(ctx, err)
+			}
+			continue
+		}
+		m.metrics[ri.JobID] = ri
+	}
+	return nil
+}
+
 func (m *batchJobMetrics) delete(jobID string) {
 	m.Lock()
 	defer m.Unlock()

cmd/batch-rotate.go

@@ -267,8 +267,12 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
+	retryAttempts := job.KeyRotate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchKeyRotateJobDefaultRetries
+	}
 	delay := job.KeyRotate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
 
@@ -354,7 +358,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		return err
 	}
 
-	retryAttempts := ri.RetryAttempts
 	ctx, cancel := context.WithCancel(ctx)
 
 	results := make(chan itemOrErr[ObjectInfo], 100)

cmd/bitrot-streaming.go

@@ -141,13 +141,7 @@ func (b *streamingBitrotReader) Close() error {
 	}
 	if closer, ok := b.rc.(io.Closer); ok {
 		// drain the body for connection reuse at network layer.
-		xhttp.DrainBody(struct {
+		xhttp.DrainBody(io.NopCloser(b.rc))
-			io.Reader
-			io.Closer
-		}{
-			Reader: b.rc,
-			Closer: closeWrapper(func() error { return nil }),
-		})
 		return closer.Close()
 	}
 	return nil

cmd/bootstrap-peer-server.go

@@ -183,11 +183,19 @@ var binaryChecksum = getBinaryChecksum()
 
 func getBinaryChecksum() string {
 	mw := md5.New()
-	b, err := os.Open(os.Args[0])
+	binPath, err := os.Executable()
-	if err == nil {
+	if err != nil {
+		logger.Error("Calculating checksum failed: %s", err)
+		return "00000000000000000000000000000000"
+	}
+	b, err := os.Open(binPath)
+	if err != nil {
+		logger.Error("Calculating checksum failed: %s", err)
+		return "00000000000000000000000000000000"
+	}
+
 	defer b.Close()
 	io.Copy(mw, b)
-	}
 	return hex.EncodeToString(mw.Sum(nil))
 }
 

cmd/bucket-handlers.go

@@ -28,6 +28,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"mime"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
@@ -53,6 +54,7 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/hash"
@@ -669,8 +671,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
-		defer globalCacheConfig.Delete(bucket, dobj.ObjectName)
-
 		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
 			// copy so we can re-add null ID.
 			dobj := dobj
@@ -889,6 +889,30 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	})
 }
 
+// multipartReader is just like https://pkg.go.dev/net/http#Request.MultipartReader but
+// rejects multipart/mixed as its not supported in S3 API.
+func multipartReader(r *http.Request) (*multipart.Reader, error) {
+	v := r.Header.Get("Content-Type")
+	if v == "" {
+		return nil, http.ErrNotMultipart
+	}
+	if r.Body == nil {
+		return nil, errors.New("missing form body")
+	}
+	d, params, err := mime.ParseMediaType(v)
+	if err != nil {
+		return nil, http.ErrNotMultipart
+	}
+	if d != "multipart/form-data" {
+		return nil, http.ErrNotMultipart
+	}
+	boundary, ok := params["boundary"]
+	if !ok {
+		return nil, http.ErrMissingBoundary
+	}
+	return multipart.NewReader(r.Body, boundary), nil
+}
+
 // PostPolicyBucketHandler - POST policy
 // ----------
 // This implementation of the POST operation handles object creation with a specified
@@ -922,9 +946,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	if r.ContentLength <= 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEmptyRequestBody), r.URL)
+		return
+	}
+
 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary files.
-	mp, err := r.MultipartReader()
+	mp, err := multipartReader(r)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
@@ -936,7 +965,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	var (
 		reader        io.Reader
-		fileSize      int64 = -1
+		actualSize    int64 = -1
 		fileName      string
 		fanOutEntries = make([]minio.PutObjectFanOutEntry, 0, 100)
 	)
@@ -944,6 +973,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	maxParts := 1000
 	// Canonicalize the form values into http.Header.
 	formValues := make(http.Header)
+	var headerLen int64
 	for {
 		part, err := mp.NextRawPart()
 		if errors.Is(err, io.EOF) {
@@ -985,7 +1015,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			return
 		}
 
-		var b bytes.Buffer
+		headerLen += int64(len(name)) + int64(len(fileName))
 		if name != "file" {
 			if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
 				dec := json.NewDecoder(part)
@@ -996,7 +1026,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					if err := dec.Decode(&m); err != nil {
 						part.Close()
 						apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-						apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
+						apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
 						writeErrorResponse(ctx, w, apiErr, r.URL)
 						return
 					}
@@ -1006,8 +1036,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				continue
 			}
 
+			buf := bytebufferpool.Get()
 			// value, store as string in memory
-			n, err := io.CopyN(&b, part, maxMemoryBytes+1)
+			n, err := io.CopyN(buf, part, maxMemoryBytes+1)
+			value := buf.String()
+			buf.Reset()
+			bytebufferpool.Put(buf)
 			part.Close()
 
 			if err != nil && err != io.EOF {
@@ -1029,7 +1063,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(ctx, w, apiErr, r.URL)
 				return
 			}
-			formValues[http.CanonicalHeaderKey(name)] = append(formValues[http.CanonicalHeaderKey(name)], b.String())
+			headerLen += n
+			formValues[http.CanonicalHeaderKey(name)] = append(formValues[http.CanonicalHeaderKey(name)], value)
 			continue
 		}
 
@@ -1038,6 +1073,21 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		// The file or text content must be the last field in the form.
 		// You cannot upload more than one file at a time.
 		reader = part
+
+		possibleShardSize := (r.ContentLength - headerLen)
+		if globalStorageClass.ShouldInline(possibleShardSize, false) { // keep versioned false for this check
+			var b bytes.Buffer
+			n, err := io.Copy(&b, reader)
+			if err != nil {
+				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+			reader = &b
+			actualSize = n
+		}
+
 		// we have found the File part of the request we are done processing multipart-form
 		break
 	}
@@ -1139,11 +1189,33 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
+	clientETag, err := etag.FromContentMD5(formValues)
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL)
+		return
+	}
+
+	var forceMD5 []byte
+	// Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag. Optionally enable this also
+	// for server that is started with `--no-compat`.
+	kind, _ := crypto.IsRequested(formValues)
+	if !etag.ContentMD5Requested(formValues) && (kind == crypto.SSEC || kind == crypto.S3KMS || !globalServerCtxt.StrictS3Compat) {
+		forceMD5 = mustGetUUIDBytes()
+	}
+
+	hashReader, err := hash.NewReaderWithOpts(ctx, reader, hash.Options{
+		Size:       actualSize,
+		MD5Hex:     clientETag.String(),
+		SHA256Hex:  "",
+		ActualSize: actualSize,
+		DisableMD5: false,
+		ForceMD5:   forceMD5,
+	})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+
 	if checksum != nil && checksum.Valid() {
 		if err = hashReader.AddChecksumNoTrailer(formValues, false); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1200,9 +1272,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
+	opts.WantChecksum = checksum
 
 	fanOutOpts := fanOutOptions{Checksum: checksum}
-
 	if crypto.Requested(formValues) {
 		if crypto.SSECopy.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
@@ -1247,8 +1319,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
+
+			wantSize := int64(-1)
+			if actualSize >= 0 {
+				info := ObjectInfo{Size: actualSize}
+				wantSize = info.EncryptedSize()
+			}
+
 			// do not try to verify encrypted content/
-			hashReader, err = hash.NewReader(ctx, reader, -1, "", "", -1)
+			hashReader, err = hash.NewReader(ctx, reader, wantSize, "", "", actualSize)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1328,7 +1407,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 						Key:   objInfo.Name,
 						Error: errs[i].Error(),
 					})
-
 					eventArgsList = append(eventArgsList, eventArgs{
 						EventName:    event.ObjectCreatedPost,
 						BucketName:   objInfo.Bucket,
@@ -1682,7 +1760,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
-	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
+	globalReplicationPool.Get().deleteResyncMetadata(ctx, bucket)
 
 	// Call site replication hook.
 	replLogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))

cmd/bucket-lifecycle-audit.go

@@ -17,7 +17,11 @@
 
 package cmd
 
-import "github.com/minio/minio/internal/bucket/lifecycle"
+import (
+	"strconv"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+)
 
 //go:generate stringer -type lcEventSrc -trimprefix lcEventSrc_ $GOFILE
 type lcEventSrc uint8
@@ -43,7 +47,7 @@ type lcAuditEvent struct {
 	source lcEventSrc
 }
 
-func (lae lcAuditEvent) Tags() map[string]interface{} {
+func (lae lcAuditEvent) Tags() map[string]string {
 	event := lae.Event
 	src := lae.source
 	const (
@@ -55,7 +59,7 @@ func (lae lcAuditEvent) Tags() map[string]interface{} {
 		ilmNewerNoncurrentVersions = "ilm-newer-noncurrent-versions"
 		ilmNoncurrentDays          = "ilm-noncurrent-days"
 	)
-	tags := make(map[string]interface{}, 5)
+	tags := make(map[string]string, 5)
 	if src > lcEventSrc_None {
 		tags[ilmSrc] = src.String()
 	}
@@ -63,7 +67,7 @@ func (lae lcAuditEvent) Tags() map[string]interface{} {
 	tags[ilmRuleID] = event.RuleID
 
 	if !event.Due.IsZero() {
-		tags[ilmDue] = event.Due
+		tags[ilmDue] = event.Due.Format(iso8601Format)
 	}
 
 	// rule with Transition/NoncurrentVersionTransition in effect
@@ -73,10 +77,10 @@ func (lae lcAuditEvent) Tags() map[string]interface{} {
 
 	// rule with NewernoncurrentVersions in effect
 	if event.NewerNoncurrentVersions > 0 {
-		tags[ilmNewerNoncurrentVersions] = event.NewerNoncurrentVersions
+		tags[ilmNewerNoncurrentVersions] = strconv.Itoa(event.NewerNoncurrentVersions)
 	}
 	if event.NoncurrentDays > 0 {
-		tags[ilmNoncurrentDays] = event.NoncurrentDays
+		tags[ilmNoncurrentDays] = strconv.Itoa(event.NoncurrentDays)
 	}
 	return tags
 }

cmd/bucket-lifecycle.go

@@ -71,7 +71,7 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
-func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string, err string) madmin.TraceInfo {
 	sz, _ := oi.GetActualSize()
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceILM,
@@ -81,18 +81,22 @@ func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event
 		Duration:  duration,
 		Path:      pathJoin(oi.Bucket, oi.Name),
 		Bytes:     sz,
-		Error:     "",
+		Error:     err,
 		Message:   getSource(4),
-		Custom:    map[string]string{"version-id": oi.VersionID},
+		Custom:    metadata,
 	}
 }
 
-func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
+func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string, err error) {
 	startTime := time.Now()
-	return func(event string) {
+	return func(event string, metadata map[string]string, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
-			globalTrace.Publish(ilmTrace(startTime, duration, oi, event))
+			e := ""
+			if err != nil {
+				e = err.Error()
+			}
+			globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata, e))
 		}
 	}
 }
@@ -348,7 +352,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 				traceFn := globalLifecycleSys.trace(oi)
 				if !oi.TransitionedObject.FreeVersion {
 					// nothing to be done
-					return
+					continue
 				}
 
 				ignoreNotFoundErr := func(err error) error {
@@ -362,7 +366,8 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
 				if ignoreNotFoundErr(err) != nil {
 					transitionLogIf(es.ctx, err)
-					return
+					traceFn(ILMFreeVersionDelete, nil, err)
+					continue
 				}
 
 				// Remove this free version
@@ -707,6 +712,11 @@ type auditTierOp struct {
 	Error            string `json:"error,omitempty"`
 }
 
+func (op auditTierOp) String() string {
+	// flattening the auditTierOp{} for audit
+	return fmt.Sprintf("tier:%s,respNS:%d,tx:%d,err:%s", op.Tier, op.TimeToResponseNS, op.OutputBytes, op.Error)
+}
+
 func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) {
 	startTime := time.Now()
 	return func(err error) {
@@ -730,7 +740,7 @@ func auditTierActions(ctx context.Context, tier string, bytes int64) func(err er
 			globalTierMetrics.logFailure(tier)
 		}
 
-		logger.GetReqInfo(ctx).AppendTags("tierStats", op)
+		logger.GetReqInfo(ctx).AppendTags("tierStats", op.String())
 	}
 }
 
@@ -998,7 +1008,7 @@ func ongoingRestoreObj() restoreObjStatus {
 	}
 }
 
-// completeRestoreObj constructs restoreObjStatus for a completed restore-object with given expiry.
+// completedRestoreObj constructs restoreObjStatus for a completed restore-object with given expiry.
 func completedRestoreObj(expiry time.Time) restoreObjStatus {
 	return restoreObjStatus{
 		ongoing: false,

cmd/bucket-metadata-sys.go

@@ -39,6 +39,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
 	"github.com/minio/pkg/v3/sync/errgroup"
+	"golang.org/x/sync/singleflight"
 )
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -47,6 +48,7 @@ type BucketMetadataSys struct {
 
 	sync.RWMutex
 	initialized bool
+	group       *singleflight.Group
 	metadataMap map[string]BucketMetadata
 }
 
@@ -62,6 +64,7 @@ func (sys *BucketMetadataSys) Count() int {
 func (sys *BucketMetadataSys) Remove(buckets ...string) {
 	sys.Lock()
 	for _, bucket := range buckets {
+		sys.group.Forget(bucket)
 		delete(sys.metadataMap, bucket)
 		globalBucketMonitor.DeleteBucket(bucket)
 	}
@@ -121,6 +124,7 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		meta.PolicyConfigUpdatedAt = updatedAt
 	case bucketNotificationConfig:
 		meta.NotificationConfigXML = configData
+		meta.NotificationConfigUpdatedAt = updatedAt
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 		meta.LifecycleConfigUpdatedAt = updatedAt
@@ -150,12 +154,13 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		if err != nil {
 			return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
+		meta.BucketTargetsConfigUpdatedAt = updatedAt
+		meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
 	default:
 		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
-	err = sys.save(ctx, meta)
+	return updatedAt, sys.save(ctx, meta)
-	return updatedAt, err
 }
 
 func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
@@ -262,6 +267,21 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 	return meta.versioningConfig, meta.VersioningConfigUpdatedAt, nil
 }
 
+// GetBucketPolicy returns configured bucket policy
+func (sys *BucketMetadataSys) GetBucketPolicy(bucket string) (*policy.BucketPolicy, time.Time, error) {
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, time.Time{}, err
+	}
+	if meta.policyConfig == nil {
+		return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
+	}
+	return meta.policyConfig, meta.PolicyConfigUpdatedAt, nil
+}
+
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
@@ -451,13 +471,20 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
 	if ok {
 		return meta, reloaded, nil
 	}
+
+	val, err, _ := sys.group.Do(bucket, func() (val interface{}, err error) {
 		meta, err = loadBucketMetadata(ctx, objAPI, bucket)
 		if err != nil {
 			if !sys.Initialized() {
 				// bucket metadata not yet initialized
-			return newBucketMetadata(bucket), reloaded, errBucketMetadataNotInitialized
+				return newBucketMetadata(bucket), errBucketMetadataNotInitialized
 			}
-		return meta, reloaded, err
+		}
+		return meta, err
+	})
+	meta, _ = val.(BucketMetadata)
+	if err != nil {
+		return meta, false, err
 	}
 	sys.Lock()
 	sys.metadataMap[bucket] = meta
@@ -480,7 +507,7 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []string, objAPI
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []string, failedBuckets map[string]struct{}) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []string) {
 	g := errgroup.WithNErrs(len(buckets))
 	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
@@ -521,10 +548,6 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
 
 	for i, meta := range bucketMetas {
 		if errs[i] != nil {
-			if failedBuckets == nil {
-				failedBuckets = make(map[string]struct{})
-			}
-			failedBuckets[buckets[i]] = struct{}{}
 			continue
 		}
 		globalEventNotifier.set(buckets[i], meta)   // set notification targets
@@ -532,7 +555,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
 	}
 }
 
-func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, failedBuckets map[string]struct{}) {
+func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
 	const bucketMetadataRefresh = 15 * time.Minute
 
 	sleeper := newDynamicSleeper(2, 150*time.Millisecond, false)
@@ -562,7 +585,10 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 			for i := range buckets {
 				wait := sleeper.Timer(ctx)
 
-				meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
+				bucket := buckets[i].Name
+				updated := false
+
+				meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket)
 				if err != nil {
 					internalLogIf(ctx, err, logger.WarningKind)
 					wait() // wait to proceed to next entry.
@@ -570,14 +596,16 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 				}
 
 				sys.Lock()
-				sys.metadataMap[buckets[i].Name] = meta
+				// Update if the bucket metadata in the memory is older than on-disk one
+				if lu := sys.metadataMap[bucket].lastUpdate(); lu.Before(meta.lastUpdate()) {
+					updated = true
+					sys.metadataMap[bucket] = meta
+				}
 				sys.Unlock()
 
-				// Initialize the failed buckets
+				if updated {
-				if _, ok := failedBuckets[buckets[i].Name]; ok {
+					globalEventNotifier.set(bucket, meta)
-					globalEventNotifier.set(buckets[i].Name, meta)
+					globalBucketTargetSys.set(bucket, meta)
-					globalBucketTargetSys.set(buckets[i].Name, meta)
-					delete(failedBuckets, buckets[i].Name)
 				}
 
 				wait() // wait to proceed to next entry.
@@ -598,13 +626,12 @@ func (sys *BucketMetadataSys) Initialized() bool {
 // Loads bucket metadata for all buckets into BucketMetadataSys.
 func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
 	count := globalEndpoints.ESCount() * 10
-	failedBuckets := make(map[string]struct{})
 	for {
 		if len(buckets) < count {
-			sys.concurrentLoad(ctx, buckets, failedBuckets)
+			sys.concurrentLoad(ctx, buckets)
 			break
 		}
-		sys.concurrentLoad(ctx, buckets[:count], failedBuckets)
+		sys.concurrentLoad(ctx, buckets[:count])
 		buckets = buckets[count:]
 	}
 
@@ -613,7 +640,7 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
 	sys.Unlock()
 
 	if globalIsDistErasure {
-		go sys.refreshBucketsMetadataLoop(ctx, failedBuckets)
+		go sys.refreshBucketsMetadataLoop(ctx)
 	}
 }
 
@@ -630,5 +657,6 @@ func (sys *BucketMetadataSys) Reset() {
 func NewBucketMetadataSys() *BucketMetadataSys {
 	return &BucketMetadataSys{
 		metadataMap: make(map[string]BucketMetadata),
+		group:       &singleflight.Group{},
 	}
 }

cmd/bucket-metadata.go

@@ -81,6 +81,7 @@ type BucketMetadata struct {
 	ReplicationConfigXML        []byte
 	BucketTargetsConfigJSON     []byte
 	BucketTargetsConfigMetaJSON []byte
+
 	PolicyConfigUpdatedAt            time.Time
 	ObjectLockConfigUpdatedAt        time.Time
 	EncryptionConfigUpdatedAt        time.Time
@@ -89,6 +90,10 @@ type BucketMetadata struct {
 	ReplicationConfigUpdatedAt       time.Time
 	VersioningConfigUpdatedAt        time.Time
 	LifecycleConfigUpdatedAt         time.Time
+	NotificationConfigUpdatedAt      time.Time
+	BucketTargetsConfigUpdatedAt     time.Time
+	BucketTargetsConfigMetaUpdatedAt time.Time
+	// Add a new UpdatedAt field and update lastUpdate function
 
 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.BucketPolicy
@@ -120,6 +125,46 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 
+// Return the last update of this bucket metadata, which
+// means, the last update of any policy document.
+func (b BucketMetadata) lastUpdate() (t time.Time) {
+	if b.PolicyConfigUpdatedAt.After(t) {
+		t = b.PolicyConfigUpdatedAt
+	}
+	if b.ObjectLockConfigUpdatedAt.After(t) {
+		t = b.ObjectLockConfigUpdatedAt
+	}
+	if b.EncryptionConfigUpdatedAt.After(t) {
+		t = b.EncryptionConfigUpdatedAt
+	}
+	if b.TaggingConfigUpdatedAt.After(t) {
+		t = b.TaggingConfigUpdatedAt
+	}
+	if b.QuotaConfigUpdatedAt.After(t) {
+		t = b.QuotaConfigUpdatedAt
+	}
+	if b.ReplicationConfigUpdatedAt.After(t) {
+		t = b.ReplicationConfigUpdatedAt
+	}
+	if b.VersioningConfigUpdatedAt.After(t) {
+		t = b.VersioningConfigUpdatedAt
+	}
+	if b.LifecycleConfigUpdatedAt.After(t) {
+		t = b.LifecycleConfigUpdatedAt
+	}
+	if b.NotificationConfigUpdatedAt.After(t) {
+		t = b.NotificationConfigUpdatedAt
+	}
+	if b.BucketTargetsConfigUpdatedAt.After(t) {
+		t = b.BucketTargetsConfigUpdatedAt
+	}
+	if b.BucketTargetsConfigMetaUpdatedAt.After(t) {
+		t = b.BucketTargetsConfigMetaUpdatedAt
+	}
+
+	return
+}
+
 // Versioning returns true if versioning is enabled
 func (b BucketMetadata) Versioning() bool {
 	return b.LockEnabled || (b.versioningConfig != nil && b.versioningConfig.Enabled()) || (b.objectLockConfig != nil && b.objectLockConfig.Enabled())
@@ -440,6 +485,18 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.LifecycleConfigUpdatedAt.IsZero() {
 		b.LifecycleConfigUpdatedAt = b.Created
 	}
+
+	if b.NotificationConfigUpdatedAt.IsZero() {
+		b.NotificationConfigUpdatedAt = b.Created
+	}
+
+	if b.BucketTargetsConfigUpdatedAt.IsZero() {
+		b.BucketTargetsConfigUpdatedAt = b.Created
+	}
+
+	if b.BucketTargetsConfigMetaUpdatedAt.IsZero() {
+		b.BucketTargetsConfigMetaUpdatedAt = b.Created
+	}
 }
 
 // Save config to supplied ObjectLayer api.

cmd/bucket-metadata_gen.go

@@ -156,6 +156,24 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 				return
 			}
+		case "NotificationConfigUpdatedAt":
+			z.NotificationConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigUpdatedAt":
+			z.BucketTargetsConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigMetaUpdatedAt":
+			z.BucketTargetsConfigMetaUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -169,9 +187,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 22
+	// map header, size 25
 	// write "Name"
-	err = en.Append(0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -390,15 +408,45 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 		return
 	}
+	// write "NotificationConfigUpdatedAt"
+	err = en.Append(0xbb, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.NotificationConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationConfigUpdatedAt")
+		return
+	}
+	// write "BucketTargetsConfigUpdatedAt"
+	err = en.Append(0xbc, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.BucketTargetsConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt")
+		return
+	}
+	// write "BucketTargetsConfigMetaUpdatedAt"
+	err = en.Append(0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.BucketTargetsConfigMetaUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 22
+	// map header, size 25
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -463,6 +511,15 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "LifecycleConfigUpdatedAt"
 	o = append(o, 0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.LifecycleConfigUpdatedAt)
+	// string "NotificationConfigUpdatedAt"
+	o = append(o, 0xbb, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.NotificationConfigUpdatedAt)
+	// string "BucketTargetsConfigUpdatedAt"
+	o = append(o, 0xbc, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.BucketTargetsConfigUpdatedAt)
+	// string "BucketTargetsConfigMetaUpdatedAt"
+	o = append(o, 0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.BucketTargetsConfigMetaUpdatedAt)
 	return
 }
 
@@ -616,6 +673,24 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 				return
 			}
+		case "NotificationConfigUpdatedAt":
+			z.NotificationConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigUpdatedAt":
+			z.BucketTargetsConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigMetaUpdatedAt":
+			z.BucketTargetsConfigMetaUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -630,6 +705,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize
 	return
 }

cmd/bucket-quota.go

@@ -50,6 +50,9 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 	bucketStorageCache.InitOnce(10*time.Second,
 		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
 		func(ctx context.Context) (DataUsageInfo, error) {
+			if objAPI == nil {
+				return DataUsageInfo{}, errServerNotInitialized
+			}
 			ctx, done := context.WithTimeout(ctx, 2*time.Second)
 			defer done()
 
@@ -59,7 +62,9 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 }
 
 // GetBucketUsageInfo return bucket usage info for a given bucket
-func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string) (BucketUsageInfo, error) {
+func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string) BucketUsageInfo {
+	sys.Init(newObjectLayerFn())
+
 	dui, err := bucketStorageCache.GetWithCtx(ctx)
 	timedout := OperationTimedOut{}
 	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
@@ -73,10 +78,10 @@ func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string
 	if len(dui.BucketsUsage) > 0 {
 		bui, ok := dui.BucketsUsage[bucket]
 		if ok {
-			return bui, nil
+			return bui
 		}
 	}
-	return BucketUsageInfo{}, nil
+	return BucketUsageInfo{}
 }
 
 // parseBucketQuota parses BucketQuota from json
@@ -118,11 +123,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 
-		bui, err := sys.GetBucketUsageInfo(ctx, bucket)
+		bui := sys.GetBucketUsageInfo(ctx, bucket)
-		if err != nil {
-			return err
-		}
-
 		if bui.Size > 0 && ((bui.Size + uint64(size)) >= quotaSize) {
 			return BucketQuotaExceeded{Bucket: bucket}
 		}

cmd/bucket-replication-handlers.go

@@ -230,7 +230,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	stats := globalReplicationStats.getLatestReplicationStats(bucket)
+	stats := globalReplicationStats.Load().getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
 	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
@@ -286,7 +286,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsV2Handler(w http.Respons
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	stats := globalReplicationStats.getLatestReplicationStats(bucket)
+	stats := globalReplicationStats.Load().getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
 	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
@@ -422,7 +422,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		return
 	}
 
-	if err := globalReplicationPool.resyncer.start(ctx, objectAPI, resyncOpts{
+	if err := globalReplicationPool.Get().resyncer.start(ctx, objectAPI, resyncOpts{
 		bucket:       bucket,
 		arn:          arn,
 		resyncID:     resetID,

cmd/bucket-replication-metrics.go

@@ -119,7 +119,7 @@ func (a *ActiveWorkerStat) update() {
 	if a == nil {
 		return
 	}
-	a.Curr = globalReplicationPool.ActiveWorkers()
+	a.Curr = globalReplicationPool.Get().ActiveWorkers()
 	a.hist.Update(int64(a.Curr))
 	a.Avg = float32(a.hist.Mean())
 	a.Max = int(a.hist.Max())

cmd/bucket-replication-stats.go

@@ -87,6 +87,9 @@ func (r *ReplicationStats) updateMovingAvg() {
 
 // ActiveWorkers returns worker stats
 func (r *ReplicationStats) ActiveWorkers() ActiveWorkerStat {
+	if r == nil {
+		return ActiveWorkerStat{}
+	}
 	r.wlock.RLock()
 	defer r.wlock.RUnlock()
 	w := r.workers.get()
@@ -351,6 +354,9 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 }
 
 func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketStats) {
+	if r == nil {
+		return nil
+	}
 	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
 	bucketsReplicationStats = make(map[string]BucketStats, len(bucketsUsage))
 
@@ -460,6 +466,9 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucket
 
 // get the most current of in-memory replication stats  and data usage info from crawler.
 func (r *ReplicationStats) getLatestReplicationStats(bucket string) (s BucketStats) {
+	if r == nil {
+		return s
+	}
 	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
 	return r.calculateBucketReplicationStats(bucket, bucketStats)
 }
@@ -495,9 +504,14 @@ func (r *ReplicationStats) decQ(bucket string, sz int64, isDelMarker bool, opTyp
 
 // incProxy increments proxy metrics for proxied calls
 func (r *ReplicationStats) incProxy(bucket string, api replProxyAPI, isErr bool) {
+	if r != nil {
 		r.pCache.inc(bucket, api, isErr)
 	}
+}
 
 func (r *ReplicationStats) getProxyStats(bucket string) ProxyMetric {
+	if r == nil {
+		return ProxyMetric{}
+	}
 	return r.pCache.getBucketStats(bucket)
 }

cmd/bucket-replication.go

@@ -50,6 +50,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/once"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
 	"golang.org/x/exp/maps"
@@ -478,7 +479,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
-		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -548,7 +549,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	// to decrement pending count later.
 	for _, rinfo := range rinfos.Targets {
 		if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
-			globalReplicationStats.Update(dobj.Bucket, rinfo, replicationStatus,
+			globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus,
 				prevStatus)
 		}
 	}
@@ -556,7 +557,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
-		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
 	if replicationStatus != prevStatus {
@@ -888,6 +889,14 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
+
+	if crypto.S3KMS.IsEncrypted(objInfo.UserDefined) {
+		sseEnc, err := encrypt.NewSSEKMS(objInfo.KMSKeyID(), nil)
+		if err != nil {
+			return putOpts, err
+		}
+		putOpts.ServerSideEncryption = sseEnc
+	}
 	return
 }
 
@@ -1054,7 +1063,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
-		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
 		return
 	}
 	ctx = lkctx.Context()
@@ -1139,7 +1148,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		for _, rinfo := range rinfos.Targets {
 			if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
 				rinfo.OpType = opType // update optype to reflect correct operation.
-				globalReplicationStats.Update(bucket, rinfo, rinfo.ReplicationStatus, rinfo.PrevReplicationStatus)
+				globalReplicationStats.Load().Update(bucket, rinfo, rinfo.ReplicationStatus, rinfo.PrevReplicationStatus)
 			}
 		}
 	}
@@ -1159,7 +1168,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		ri.EventType = ReplicateMRF
 		ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
 		ri.RetryCount++
-		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
 	}
 }
 
@@ -1641,7 +1650,6 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 
 	var (
 		hr     *hash.Reader
-		pInfo  minio.ObjectPart
 		isSSEC = crypto.SSEC.IsEncrypted(objInfo.UserDefined)
 	)
 
@@ -1668,18 +1676,19 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			CustomHeader: cHeader,
 		}
 
+		var size int64
 		if isSSEC {
-			objectSize += partInfo.Size
+			size = partInfo.Size
-			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts)
 		} else {
-			objectSize += partInfo.ActualSize
+			size = partInfo.ActualSize
-			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
 		}
+		objectSize += size
+		pInfo, err := c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, size, popts)
 		if err != nil {
 			return err
 		}
-		if !isSSEC && pInfo.Size != partInfo.ActualSize {
+		if pInfo.Size != size {
-			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
+			return fmt.Errorf("ssec(%t): Part size mismatch: got %d, want %d", isSSEC, pInfo.Size, size)
 		}
 		uploadedParts = append(uploadedParts, minio.CompletePart{
 			PartNumber:     pInfo.PartNumber,
@@ -1691,7 +1700,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		})
 	}
 	userMeta := map[string]string{
-		validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"],
+		xhttp.MinIOReplicationActualObjectSize: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"],
 	}
 	if isSSEC && objInfo.UserDefined[ReplicationSsecChecksumHeader] != "" {
 		userMeta[ReplicationSsecChecksumHeader] = objInfo.UserDefined[ReplicationSsecChecksumHeader]
@@ -1787,8 +1796,8 @@ const (
 )
 
 var (
-	globalReplicationPool  *ReplicationPool
+	globalReplicationPool  = once.NewSingleton[ReplicationPool]()
-	globalReplicationStats *ReplicationStats
+	globalReplicationStats atomic.Pointer[ReplicationStats]
 )
 
 // ReplicationPool describes replication pool
@@ -1803,6 +1812,7 @@ type ReplicationPool struct {
 	priority    string
 	maxWorkers  int
 	maxLWorkers int
+	stats       *ReplicationStats
 
 	mu       sync.RWMutex
 	mrfMU    sync.Mutex
@@ -1849,7 +1859,7 @@ const (
 )
 
 // NewReplicationPool creates a pool of replication workers of specified size
-func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts) *ReplicationPool {
+func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts, stats *ReplicationStats) *ReplicationPool {
 	var workers, failedWorkers int
 	priority := "auto"
 	maxWorkers := WorkerMaxLimit
@@ -1891,6 +1901,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		mrfStopCh:       make(chan struct{}, 1),
 		ctx:             ctx,
 		objLayer:        o,
+		stats:           stats,
 		priority:        priority,
 		maxWorkers:      maxWorkers,
 		maxLWorkers:     maxLWorkers,
@@ -1918,11 +1929,11 @@ func (p *ReplicationPool) AddMRFWorker() {
 			}
 			switch v := oi.(type) {
 			case ReplicateObjectInfo:
-				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				atomic.AddInt32(&p.activeMRFWorkers, 1)
 				replicateObject(p.ctx, v, p.objLayer)
 				atomic.AddInt32(&p.activeMRFWorkers, -1)
-				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 
 			default:
 				bugLogIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type")
@@ -1950,9 +1961,9 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, 1)
 				}
-				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				replicateObject(p.ctx, v, p.objLayer)
-				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, -1)
 				}
@@ -1960,10 +1971,10 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, 1)
 				}
-				globalReplicationStats.incQ(v.Bucket, 0, true, v.OpType)
+				p.stats.incQ(v.Bucket, 0, true, v.OpType)
 
 				replicateDelete(p.ctx, v, p.objLayer)
-				globalReplicationStats.decQ(v.Bucket, 0, true, v.OpType)
+				p.stats.decQ(v.Bucket, 0, true, v.OpType)
 
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, -1)
@@ -1990,9 +2001,9 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, 1)
 				}
-				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				replicateObject(p.ctx, v, p.objLayer)
-				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, -1)
 				}
@@ -2156,7 +2167,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		case <-p.ctx.Done():
 		case p.lrgworkers[h%uint64(len(p.lrgworkers))] <- ri:
 		default:
-			globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+			p.queueMRFSave(ri.ToMRFEntry())
 			p.mu.RLock()
 			maxLWorkers := p.maxLWorkers
 			existing := len(p.lrgworkers)
@@ -2187,7 +2198,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	case healCh <- ri:
 	case ch <- ri:
 	default:
-		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
 		maxWorkers := p.maxWorkers
@@ -2223,7 +2234,7 @@ func queueReplicateDeletesWrapper(doi DeletedObjectReplicationInfo, existingObje
 			doi.ResetID = v.ResetID
 			doi.TargetArn = k
 
-			globalReplicationPool.queueReplicaDeleteTask(doi)
+			globalReplicationPool.Get().queueReplicaDeleteTask(doi)
 		}
 	}
 }
@@ -2244,7 +2255,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 	case <-p.ctx.Done():
 	case ch <- doi:
 	default:
-		globalReplicationPool.queueMRFSave(doi.ToMRFEntry())
+		p.queueMRFSave(doi.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
 		maxWorkers := p.maxWorkers
@@ -2274,9 +2285,10 @@ type replicationPoolOpts struct {
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
-	globalReplicationPool = NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts())
+	stats := NewReplicationStats(ctx, objectAPI)
-	globalReplicationStats = NewReplicationStats(ctx, objectAPI)
+	globalReplicationPool.Set(NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts(), stats))
-	go globalReplicationStats.trackEWMA()
+	globalReplicationStats.Store(stats)
+	go stats.trackEWMA()
 }
 
 type proxyResult struct {
@@ -2482,7 +2494,7 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
 	if dsc.Synchronous() {
 		replicateObject(ctx, ri, o)
 	} else {
-		globalReplicationPool.queueReplicaTask(ri)
+		globalReplicationPool.Get().queueReplicaTask(ri)
 	}
 }
 
@@ -2610,9 +2622,9 @@ func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts
 }
 
 func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) {
-	globalReplicationPool.queueReplicaDeleteTask(dv)
+	globalReplicationPool.Get().queueReplicaDeleteTask(dv)
 	for arn := range dv.ReplicationState.Targets {
-		globalReplicationStats.Update(dv.Bucket, replicatedTargetInfo{Arn: arn, Size: 0, Duration: 0, OpType: replication.DeleteReplicationType}, replication.Pending, replication.StatusType(""))
+		globalReplicationStats.Load().Update(dv.Bucket, replicatedTargetInfo{Arn: arn, Size: 0, Duration: 0, OpType: replication.DeleteReplicationType}, replication.Pending, replication.StatusType(""))
 	}
 }
 
@@ -3042,9 +3054,9 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 	if len(tgtArns) == 0 {
 		return fmt.Errorf("arn %s specified for resync not found in replication config", opts.arn)
 	}
-	globalReplicationPool.resyncer.RLock()
+	globalReplicationPool.Get().resyncer.RLock()
-	data, ok := globalReplicationPool.resyncer.statusMap[opts.bucket]
+	data, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
-	globalReplicationPool.resyncer.RUnlock()
+	globalReplicationPool.Get().resyncer.RUnlock()
 	if !ok {
 		data, err = loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
 		if err != nil {
@@ -3070,9 +3082,9 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 		return err
 	}
 
-	globalReplicationPool.resyncer.Lock()
+	globalReplicationPool.Get().resyncer.Lock()
-	defer globalReplicationPool.resyncer.Unlock()
+	defer globalReplicationPool.Get().resyncer.Unlock()
-	brs, ok := globalReplicationPool.resyncer.statusMap[opts.bucket]
+	brs, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
 	if !ok {
 		brs = BucketReplicationResyncStatus{
 			Version:    resyncMetaVersion,
@@ -3080,8 +3092,8 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 		}
 	}
 	brs.TargetsMap[opts.arn] = status
-	globalReplicationPool.resyncer.statusMap[opts.bucket] = brs
+	globalReplicationPool.Get().resyncer.statusMap[opts.bucket] = brs
-	go globalReplicationPool.resyncer.resyncBucket(GlobalContext, objAPI, false, opts)
+	go globalReplicationPool.Get().resyncer.resyncBucket(GlobalContext, objAPI, false, opts)
 	return nil
 }
 
@@ -3413,7 +3425,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 		if roi.ReplicationStatus == replication.Pending ||
 			roi.ReplicationStatus == replication.Failed ||
 			roi.VersionPurgeStatus == Failed || roi.VersionPurgeStatus == Pending {
-			globalReplicationPool.queueReplicaDeleteTask(dv)
+			globalReplicationPool.Get().queueReplicaDeleteTask(dv)
 			return
 		}
 		// if replication status is Complete on DeleteMarker and existing object resync required
@@ -3429,12 +3441,12 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 	switch roi.ReplicationStatus {
 	case replication.Pending, replication.Failed:
 		roi.EventType = ReplicateHeal
-		globalReplicationPool.queueReplicaTask(roi)
+		globalReplicationPool.Get().queueReplicaTask(roi)
 		return
 	}
 	if roi.ExistingObjResync.mustResync() {
 		roi.EventType = ReplicateExisting
-		globalReplicationPool.queueReplicaTask(roi)
+		globalReplicationPool.Get().queueReplicaTask(roi)
 	}
 	return
 }
@@ -3499,8 +3511,8 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 		return
 	}
 	if entry.RetryCount > mrfRetryLimit { // let scanner catch up if retry count exceeded
-		atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedCount, 1)
+		atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
-		atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
+		atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
 		return
 	}
 
@@ -3513,8 +3525,8 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 		select {
 		case p.mrfSaveCh <- entry:
 		default:
-			atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedCount, 1)
+			atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
-			atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
+			atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
 		}
 	}
 }
@@ -3563,7 +3575,7 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
 	if !p.initialized() {
 		return
 	}
-	atomic.StoreUint64(&globalReplicationStats.mrfStats.LastFailedCount, uint64(len(entries)))
+	atomic.StoreUint64(&p.stats.mrfStats.LastFailedCount, uint64(len(entries)))
 	if len(entries) == 0 {
 		return
 	}

cmd/bucket-stats.go

@@ -310,7 +310,12 @@ type ReplQNodeStats struct {
 func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats) {
 	qs.NodeName = globalLocalNodeName
 	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
-	qs.ActiveWorkers = globalReplicationStats.ActiveWorkers()
+	grs := globalReplicationStats.Load()
+	if grs != nil {
+		qs.ActiveWorkers = grs.ActiveWorkers()
+	} else {
+		qs.ActiveWorkers = ActiveWorkerStat{}
+	}
 	qs.XferStats = make(map[RMetricName]XferStats)
 	qs.QStats = r.qCache.getBucketStats(bucket)
 	qs.TgtXferStats = make(map[string]map[RMetricName]XferStats)
@@ -402,7 +407,7 @@ func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats)
 func (r *ReplicationStats) getNodeQueueStatsSummary() (qs ReplQNodeStats) {
 	qs.NodeName = globalLocalNodeName
 	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
-	qs.ActiveWorkers = globalReplicationStats.ActiveWorkers()
+	qs.ActiveWorkers = globalReplicationStats.Load().ActiveWorkers()
 	qs.XferStats = make(map[RMetricName]XferStats)
 	qs.QStats = r.qCache.getSiteStats()
 	qs.MRFStats = ReplicationMRFStats{

cmd/common-main.go

@@ -91,9 +91,6 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
-	go globalBatchJobsMetrics.purgeJobMetrics()
-
 	t, _ := minioVersionToReleaseTime(Version)
 	if !t.IsZero() {
 		globalVersionUnix = uint64(t.Unix())
@@ -138,6 +135,9 @@ func minioConfigToConsoleFeatures() {
 	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
 		os.Setenv("CONSOLE_BROWSER_REDIRECT_URL", value)
 	}
+	if value := env.Get(config.EnvConsoleDebugLogLevel, ""); value != "" {
+		os.Setenv("CONSOLE_DEBUG_LOGLEVEL", value)
+	}
 	// pass the console subpath configuration
 	if globalBrowserRedirectURL != nil {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
@@ -461,6 +461,7 @@ func handleCommonArgs(ctxt serverCtxt) {
 	certsDir := ctxt.CertsDir
 	certsSet := ctxt.certsDirSet
 
+	if globalBrowserEnabled {
 		if consoleAddr == "" {
 			p, err := xnet.GetFreePort()
 			if err != nil {
@@ -481,6 +482,7 @@ func handleCommonArgs(ctxt serverCtxt) {
 		if consoleAddr == addr {
 			logger.FatalIf(errors.New("--console-address cannot be same as --address"), "Unable to start the server")
 		}
+	}
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(addr)
 	if globalMinioPort == "0" {
@@ -492,7 +494,9 @@ func handleCommonArgs(ctxt serverCtxt) {
 		globalDynamicAPIPort = true
 	}
 
+	if globalBrowserEnabled {
 		globalMinioConsoleHost, globalMinioConsolePort = mustSplitHostPort(consoleAddr)
+	}
 
 	if globalMinioPort == globalMinioConsolePort {
 		logger.FatalIf(errors.New("--console-address port cannot be same as --address port"), "Unable to start the server")
@@ -696,12 +700,16 @@ func loadEnvVarsFromFiles() {
 	}
 }
 
-func serverHandleEnvVars() {
+func serverHandleEarlyEnvVars() {
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
+}
+
+func serverHandleEnvVars() {
+	var err error
 	if globalBrowserEnabled {
 		if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
 			u, err := xnet.ParseHTTPURL(redirectURL)
@@ -758,7 +766,7 @@ func serverHandleEnvVars() {
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
 			if _, ok := dns2.IsDomainName(domainName); !ok {
-				logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+				logger.Fatal(config.ErrInvalidDomainValue(nil).Msgf("Unknown value `%s`", domainName),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
@@ -767,7 +775,7 @@ func serverHandleEnvVars() {
 		lcpSuf := lcpSuffix(globalDomainNames)
 		for _, domainName := range globalDomainNames {
 			if domainName == lcpSuf && len(globalDomainNames) > 1 {
-				logger.Fatal(config.ErrOverlappingDomainValue(nil).Msg("Overlapping domains `%s` not allowed", globalDomainNames),
+				logger.Fatal(config.ErrOverlappingDomainValue(nil).Msgf("Overlapping domains `%s` not allowed", globalDomainNames),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 		}

cmd/config-current.go

@@ -34,7 +34,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
 	"github.com/minio/minio/internal/config/batch"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
@@ -80,7 +79,6 @@ func initHelp() {
 		config.CallhomeSubSys:       callhome.DefaultKVS,
 		config.DriveSubSys:          drive.DefaultKVS,
 		config.ILMSubSys:            ilm.DefaultKVS,
-		config.CacheSubSys:          cache.DefaultKVS,
 		config.BatchSubSys:          batch.DefaultKVS,
 		config.BrowserSubSys:        browser.DefaultKVS,
 	}
@@ -229,12 +227,6 @@ func initHelp() {
 			Key:         config.EtcdSubSys,
 			Description: "persist IAM assets externally to etcd",
 		},
-		config.HelpKV{
-			Key:         config.CacheSubSys,
-			Type:        "string",
-			Description: "enable cache plugin on MinIO for GET/HEAD requests",
-			Optional:    true,
-		},
 		config.HelpKV{
 			Key:         config.BrowserSubSys,
 			Description: "manage Browser HTTP specific features, such as Security headers, etc.",
@@ -291,7 +283,6 @@ func initHelp() {
 		config.SubnetSubSys:         subnet.HelpSubnet,
 		config.CallhomeSubSys:       callhome.HelpCallhome,
 		config.DriveSubSys:          drive.HelpDrive,
-		config.CacheSubSys:          cache.Help,
 		config.BrowserSubSys:        browser.Help,
 		config.ILMSubSys:            ilm.Help,
 	}
@@ -409,10 +400,6 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
 		if _, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]); err != nil {
 			return err
 		}
-	case config.CacheSubSys:
-		if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default], globalRemoteTargetTransport); err != nil {
-			return err
-		}
 	case config.PolicyOPASubSys:
 		// In case legacy OPA config is being set, we treat it as if the
 		// AuthZPlugin is being set.
@@ -710,13 +697,6 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				configLogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err))
 			}
 		}
-	case config.CacheSubSys:
-		cacheCfg, err := cache.LookupConfig(s[config.CacheSubSys][config.Default], globalRemoteTargetTransport)
-		if err != nil {
-			configLogIf(ctx, fmt.Errorf("Unable to load cache config: %w", err))
-		} else {
-			globalCacheConfig.Update(cacheCfg)
-		}
 	case config.BrowserSubSys:
 		browserCfg, err := browser.LookupConfig(s[config.BrowserSubSys][config.Default])
 		if err != nil {

cmd/data-scanner.go

@@ -569,7 +569,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				APIName: "Scanner",
 				Bucket:  f.root,
 				Object:  prefixName,
-				Tags: map[string]interface{}{
+				Tags: map[string]string{
 					"x-minio-prefixes-total": strconv.Itoa(totalFolders),
 				},
 			})
@@ -1138,7 +1138,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			APIName: "Scanner",
 			Bucket:  i.bucket,
 			Object:  i.objectPath(),
-			Tags: map[string]interface{}{
+			Tags: map[string]string{
 				"x-minio-versions": strconv.Itoa(len(objInfos)),
 			},
 		})
@@ -1170,7 +1170,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			APIName: "Scanner",
 			Bucket:  i.bucket,
 			Object:  i.objectPath(),
-			Tags: map[string]interface{}{
+			Tags: map[string]string{
 				"x-minio-versions-count": strconv.Itoa(len(objInfos)),
 				"x-minio-versions-size":  strconv.FormatInt(cumulativeSize, 10),
 			},
@@ -1325,10 +1325,13 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			traceFn(ILMExpiry, nil, nil)
 			return false
 		}
 		// Assume it is still there.
-		ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
+		err := fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err)
+		ilmLogOnceIf(ctx, err, "non-transition-expiry"+obj.Name)
+		traceFn(ILMExpiry, nil, err)
 		return false
 	}
 	if dobj.Name == "" {
@@ -1336,6 +1339,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	}
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+	tags["version-id"] = dobj.VersionID
+
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
 
@@ -1547,7 +1552,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]interface{}, traceFn func(event string)) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1565,5 +1570,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
 		VersionID: oi.VersionID,
 		Tags:      tags,
 	})
-	traceFn(event)
+	traceFn(event, tags, nil)
 }

cmd/data-usage-cache.go

@@ -39,6 +39,8 @@ import (
 	"github.com/valyala/bytebufferpool"
 )
 
+//msgp:clearomitted
+
 //go:generate msgp -file $GOFILE -unexported
 
 // dataUsageHash is the hash type used.
@@ -63,7 +65,6 @@ type dataUsageEntry struct {
 	DeleteMarkers uint64            `msg:"dms"`
 	ObjSizes      sizeHistogram     `msg:"szs"`
 	ObjVersions   versionsHistogram `msg:"vh"`
-	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
 	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
 	Compacted     bool              `msg:"c"`
 }
@@ -135,93 +136,6 @@ func (ts tierStats) add(u tierStats) tierStats {
 	}
 }
 
-//msgp:tuple replicationStatsV1
-type replicationStatsV1 struct {
-	PendingSize          uint64
-	ReplicatedSize       uint64
-	FailedSize           uint64
-	ReplicaSize          uint64
-	FailedCount          uint64
-	PendingCount         uint64
-	MissedThresholdSize  uint64
-	AfterThresholdSize   uint64
-	MissedThresholdCount uint64
-	AfterThresholdCount  uint64
-}
-
-func (rsv1 replicationStatsV1) Empty() bool {
-	return rsv1.ReplicatedSize == 0 &&
-		rsv1.FailedSize == 0 &&
-		rsv1.FailedCount == 0
-}
-
-//msgp:tuple replicationStats
-type replicationStats struct {
-	PendingSize          uint64
-	ReplicatedSize       uint64
-	FailedSize           uint64
-	FailedCount          uint64
-	PendingCount         uint64
-	MissedThresholdSize  uint64
-	AfterThresholdSize   uint64
-	MissedThresholdCount uint64
-	AfterThresholdCount  uint64
-	ReplicatedCount      uint64
-}
-
-func (rs replicationStats) Empty() bool {
-	return rs.ReplicatedSize == 0 &&
-		rs.FailedSize == 0 &&
-		rs.FailedCount == 0
-}
-
-type replicationAllStats struct {
-	Targets      map[string]replicationStats `msg:"t,omitempty"`
-	ReplicaSize  uint64                      `msg:"r,omitempty"`
-	ReplicaCount uint64                      `msg:"rc,omitempty"`
-}
-
-//msgp:tuple replicationAllStatsV1
-type replicationAllStatsV1 struct {
-	Targets      map[string]replicationStats
-	ReplicaSize  uint64 `msg:"ReplicaSize,omitempty"`
-	ReplicaCount uint64 `msg:"ReplicaCount,omitempty"`
-}
-
-// empty returns true if the replicationAllStats is empty (contains no entries).
-func (r *replicationAllStats) empty() bool {
-	if r == nil {
-		return true
-	}
-	if r.ReplicaSize != 0 || r.ReplicaCount != 0 {
-		return false
-	}
-	for _, v := range r.Targets {
-		if !v.Empty() {
-			return false
-		}
-	}
-	return true
-}
-
-// clone creates a deep-copy clone.
-func (r *replicationAllStats) clone() *replicationAllStats {
-	if r == nil {
-		return nil
-	}
-
-	// Shallow copy
-	dst := *r
-
-	// Copy individual targets.
-	dst.Targets = make(map[string]replicationStats, len(r.Targets))
-	for k, v := range r.Targets {
-		dst.Targets[k] = v
-	}
-
-	return &dst
-}
-
 //msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 //msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
@@ -238,10 +152,6 @@ type dataUsageEntryV2 struct {
 type dataUsageEntryV3 struct {
 	// These fields do no include any children.
 	Size     int64
-	ReplicatedSize         uint64
-	ReplicationPendingSize uint64
-	ReplicationFailedSize  uint64
-	ReplicaSize            uint64
 	Objects  uint64
 	ObjSizes sizeHistogram
 	Children dataUsageHashMap
@@ -254,7 +164,6 @@ type dataUsageEntryV4 struct {
 	Size     int64
 	Objects  uint64
 	ObjSizes sizeHistogram
-	ReplicationStats replicationStatsV1
 }
 
 //msgp:tuple dataUsageEntryV5
@@ -265,7 +174,6 @@ type dataUsageEntryV5 struct {
 	Objects   uint64
 	Versions  uint64 // Versions that are not delete markers.
 	ObjSizes  sizeHistogram
-	ReplicationStats *replicationStatsV1
 	Compacted bool
 }
 
@@ -277,7 +185,6 @@ type dataUsageEntryV6 struct {
 	Objects   uint64
 	Versions  uint64 // Versions that are not delete markers.
 	ObjSizes  sizeHistogram
-	ReplicationStats *replicationAllStatsV1
 	Compacted bool
 }
 
@@ -290,7 +197,6 @@ type dataUsageEntryV7 struct {
 	DeleteMarkers uint64            `msg:"dms"`
 	ObjSizes      sizeHistogramV1   `msg:"szs"`
 	ObjVersions   versionsHistogram `msg:"vh"`
-	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
 	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
 	Compacted     bool              `msg:"c"`
 }
@@ -373,29 +279,6 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.ObjSizes.add(summary.totalSize)
 	e.ObjVersions.add(summary.versions)
 
-	if e.ReplicationStats == nil {
-		e.ReplicationStats = &replicationAllStats{
-			Targets: make(map[string]replicationStats),
-		}
-	} else if e.ReplicationStats.Targets == nil {
-		e.ReplicationStats.Targets = make(map[string]replicationStats)
-	}
-	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
-	e.ReplicationStats.ReplicaCount += uint64(summary.replicaCount)
-
-	for arn, st := range summary.replTargetStats {
-		tgtStat, ok := e.ReplicationStats.Targets[arn]
-		if !ok {
-			tgtStat = replicationStats{}
-		}
-		tgtStat.PendingSize += uint64(st.pendingSize)
-		tgtStat.FailedSize += uint64(st.failedSize)
-		tgtStat.ReplicatedSize += uint64(st.replicatedSize)
-		tgtStat.ReplicatedCount += uint64(st.replicatedCount)
-		tgtStat.FailedCount += st.failedCount
-		tgtStat.PendingCount += st.pendingCount
-		e.ReplicationStats.Targets[arn] = tgtStat
-	}
 	if len(summary.tiers) != 0 {
 		if e.AllTierStats == nil {
 			e.AllTierStats = newAllTierStats()
@@ -410,26 +293,6 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Versions += other.Versions
 	e.DeleteMarkers += other.DeleteMarkers
 	e.Size += other.Size
-	if other.ReplicationStats != nil {
-		if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
-		} else if e.ReplicationStats.Targets == nil {
-			e.ReplicationStats.Targets = make(map[string]replicationStats)
-		}
-		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
-		e.ReplicationStats.ReplicaCount += other.ReplicationStats.ReplicaCount
-		for arn, stat := range other.ReplicationStats.Targets {
-			st := e.ReplicationStats.Targets[arn]
-			e.ReplicationStats.Targets[arn] = replicationStats{
-				PendingSize:     stat.PendingSize + st.PendingSize,
-				FailedSize:      stat.FailedSize + st.FailedSize,
-				ReplicatedSize:  stat.ReplicatedSize + st.ReplicatedSize,
-				PendingCount:    stat.PendingCount + st.PendingCount,
-				FailedCount:     stat.FailedCount + st.FailedCount,
-				ReplicatedCount: stat.ReplicatedCount + st.ReplicatedCount,
-			}
-		}
-	}
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
@@ -490,10 +353,7 @@ func (e dataUsageEntry) clone() dataUsageEntry {
 		}
 		e.Children = ch
 	}
-	if e.ReplicationStats != nil {
+
-		// Clone ReplicationStats
-		e.ReplicationStats = e.ReplicationStats.clone()
-	}
 	if e.AllTierStats != nil {
 		e.AllTierStats = e.AllTierStats.clone()
 	}
@@ -931,22 +791,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 			ObjectSizesHistogram:    flat.ObjSizes.toMap(),
 			ObjectVersionsHistogram: flat.ObjVersions.toMap(),
 		}
-		if flat.ReplicationStats != nil {
-			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
-			bui.ReplicaCount = flat.ReplicationStats.ReplicaCount
-
-			bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
-			for arn, stat := range flat.ReplicationStats.Targets {
-				bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
-					ReplicationPendingSize:  stat.PendingSize,
-					ReplicatedSize:          stat.ReplicatedSize,
-					ReplicationFailedSize:   stat.FailedSize,
-					ReplicationPendingCount: stat.PendingCount,
-					ReplicationFailedCount:  stat.FailedCount,
-					ReplicatedCount:         stat.ReplicatedCount,
-				}
-			}
-		}
 		dst[bucket.Name] = bui
 	}
 	return dst
@@ -959,9 +803,6 @@ func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
 		return root
 	}
 	flat := d.flatten(*root)
-	if flat.ReplicationStats.empty() {
-		flat.ReplicationStats = nil
-	}
 	return &flat
 }
 
@@ -1238,20 +1079,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			if v.ReplicatedSize > 0 || v.ReplicaSize > 0 || v.ReplicationFailedSize > 0 || v.ReplicationPendingSize > 0 {
-				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
-				if cfg != nil && cfg.RoleArn != "" {
-					due.ReplicationStats = &replicationAllStats{
-						Targets: make(map[string]replicationStats),
-					}
-					due.ReplicationStats.ReplicaSize = v.ReplicaSize
-					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
-						ReplicatedSize: v.ReplicatedSize,
-						FailedSize:     v.ReplicationFailedSize,
-						PendingSize:    v.ReplicationPendingSize,
-					}
-				}
-			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
@@ -1277,36 +1104,10 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			empty := replicationStatsV1{}
-
-			if v.ReplicationStats != empty {
-				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
-				if cfg != nil && cfg.RoleArn != "" {
-					due.ReplicationStats = &replicationAllStats{
-						Targets: make(map[string]replicationStats),
-					}
-					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
-						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
-						FailedSize:     v.ReplicationStats.FailedSize,
-						FailedCount:    v.ReplicationStats.FailedCount,
-						PendingSize:    v.ReplicationStats.PendingSize,
-						PendingCount:   v.ReplicationStats.PendingCount,
-					}
-					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
-				}
-			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
 		}
-
-		// Populate compacted value and remove unneeded replica stats.
-		for k, e := range d.Cache {
-			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
-				e.ReplicationStats = nil
-			}
-			d.Cache[k] = e
-		}
 		return nil
 	case dataUsageCacheVerV5:
 		// Zstd compressed.
@@ -1328,36 +1129,10 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			if v.ReplicationStats != nil && !v.ReplicationStats.Empty() {
-				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
-				if cfg != nil && cfg.RoleArn != "" {
-					due.ReplicationStats = &replicationAllStats{
-						Targets: make(map[string]replicationStats),
-					}
-					d.Info.replication = replicationConfig{Config: cfg}
-
-					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
-						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
-						FailedSize:     v.ReplicationStats.FailedSize,
-						FailedCount:    v.ReplicationStats.FailedCount,
-						PendingSize:    v.ReplicationStats.PendingSize,
-						PendingCount:   v.ReplicationStats.PendingCount,
-					}
-					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
-				}
-			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
 		}
-
-		// Populate compacted value and remove unneeded replica stats.
-		for k, e := range d.Cache {
-			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
-				e.ReplicationStats = nil
-			}
-			d.Cache[k] = e
-		}
 		return nil
 	case dataUsageCacheVerV6:
 		// Zstd compressed.
@@ -1373,21 +1148,12 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		d.Info = dold.Info
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
-			var replicationStats *replicationAllStats
-			if v.ReplicationStats != nil {
-				replicationStats = &replicationAllStats{
-					Targets:      v.ReplicationStats.Targets,
-					ReplicaSize:  v.ReplicationStats.ReplicaSize,
-					ReplicaCount: v.ReplicationStats.ReplicaCount,
-				}
-			}
 			due := dataUsageEntry{
 				Children:  v.Children,
 				Size:      v.Size,
 				Objects:   v.Objects,
 				Versions:  v.Versions,
 				ObjSizes:  v.ObjSizes,
-				ReplicationStats: replicationStats,
 				Compacted: v.Compacted,
 			}
 			d.Cache[k] = due
@@ -1415,7 +1181,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				Objects:   v.Objects,
 				Versions:  v.Versions,
 				ObjSizes:  szHist,
-				ReplicationStats: v.ReplicationStats,
 				Compacted: v.Compacted,
 			}
 		}

cmd/data-usage-cache_gen_test.go

@@ -519,458 +519,6 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalreplicationAllStats(t *testing.T) {
-	v := replicationAllStats{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationAllStats(t *testing.T) {
-	v := replicationAllStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationAllStats Msgsize() is inaccurate")
-	}
-
-	vn := replicationAllStats{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalreplicationAllStatsV1(t *testing.T) {
-	v := replicationAllStatsV1{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationAllStatsV1(t *testing.T) {
-	v := replicationAllStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationAllStatsV1 Msgsize() is inaccurate")
-	}
-
-	vn := replicationAllStatsV1{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalreplicationStats(t *testing.T) {
-	v := replicationStats{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationStats(b *testing.B) {
-	v := replicationStats{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationStats(b *testing.B) {
-	v := replicationStats{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationStats(b *testing.B) {
-	v := replicationStats{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationStats(t *testing.T) {
-	v := replicationStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationStats Msgsize() is inaccurate")
-	}
-
-	vn := replicationStats{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationStats(b *testing.B) {
-	v := replicationStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationStats(b *testing.B) {
-	v := replicationStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalreplicationStatsV1(t *testing.T) {
-	v := replicationStatsV1{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationStatsV1(t *testing.T) {
-	v := replicationStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationStatsV1 Msgsize() is inaccurate")
-	}
-
-	vn := replicationStatsV1{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
 	v := sizeHistogram{}
 	bts, err := v.MarshalMsg(nil)

cmd/data-usage_test.go

@@ -587,17 +587,6 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		t.Fatal(err)
 	}
 	e := want.find("abucket/dir2")
-	e.ReplicationStats = &replicationAllStats{
-		Targets: map[string]replicationStats{
-			"arn": {
-				PendingSize:    1,
-				ReplicatedSize: 2,
-				FailedSize:     3,
-				FailedCount:    5,
-				PendingCount:   6,
-			},
-		},
-	}
 	want.replace("abucket/dir2", "", *e)
 	var buf bytes.Buffer
 	err = want.serializeTo(&buf)

cmd/dummy-handlers.go

@@ -44,7 +44,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketWebsite if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketAccelerate if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -109,7 +109,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketRequestPaymentHandler if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketLoggingHandler if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -181,9 +181,7 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone {
-	// we are simply re-purposing the bucketPolicyAction.
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
@@ -197,3 +195,63 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 
 	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL)
 }
+
+// PutBucketCorsHandler - PUT bucket cors, a dummy api
+func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketCors")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+}
+
+// DeleteBucketCorsHandler - DELETE bucket cors, a dummy api
+func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketCors")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+}

cmd/encryption-v1.go

@@ -607,22 +607,38 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint3
 		partEncRelOffset: partEncRelOffset,
 		parts:            oi.Parts,
 		partIndex:        partStart,
-		header:            h,
-		bucket:            bucket,
-		object:            object,
-		customerKeyHeader: h.Get(xhttp.AmzServerSideEncryptionCustomerKey),
-		copySource:        copySource,
-		metadata:          cloneMSS(oi.UserDefined),
 	}
 
-	if w.copySource {
+	// In case of SSE-C, we have to decrypt the OEK using the client-provided key.
-		w.customerKeyHeader = h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey)
+	// In case of a SSE-C server-side copy, the client might provide two keys,
+	// one for the source and one for the target. This reader is the source.
+	var ssecClientKey []byte
+	if crypto.SSEC.IsEncrypted(oi.UserDefined) {
+		if copySource && crypto.SSECopy.IsRequested(h) {
+			key, err := crypto.SSECopy.ParseHTTP(h)
+			if err != nil {
+				return nil, err
 			}
+			ssecClientKey = key[:]
+		} else {
+			key, err := crypto.SSEC.ParseHTTP(h)
+			if err != nil {
+				return nil, err
+			}
+			ssecClientKey = key[:]
+		}
+	}
+
+	// Decrypt the OEK once and reuse it for all subsequent parts.
+	objectEncryptionKey, err := decryptObjectMeta(ssecClientKey, bucket, object, oi.UserDefined)
+	if err != nil {
+		return nil, err
+	}
+	w.objectEncryptionKey = objectEncryptionKey
 
 	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
 		return nil, err
 	}
-
 	return w, nil
 }
 
@@ -639,47 +655,16 @@ type DecryptBlocksReader struct {
 	partIndex int
 	// Parts information
 	parts []ObjectPartInfo
-	header         http.Header
-	bucket, object string
-	metadata       map[string]string
 
+	objectEncryptionKey                []byte
 	partDecRelOffset, partEncRelOffset int64
-
-	copySource bool
-	// Customer Key
-	customerKeyHeader string
 }
 
 func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
-	m := cloneMSS(d.metadata)
-	// Initialize the first decrypter; new decrypters will be
-	// initialized in Read() operation as needed.
-	var key []byte
-	var err error
-	if d.copySource {
-		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(xhttp.AmzServerSideEncryptionCopyCustomerKey, d.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
-		}
-	} else {
-		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(xhttp.AmzServerSideEncryptionCustomerKey, d.customerKeyHeader)
-			key, err = ParseSSECustomerHeader(d.header)
-		}
-	}
-	if err != nil {
-		return err
-	}
-
-	objectEncryptionKey, err := decryptObjectMeta(key, d.bucket, d.object, m)
-	if err != nil {
-		return err
-	}
-
 	var partIDbin [4]byte
 	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
 
-	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+	mac := hmac.New(sha256.New, d.objectEncryptionKey) // derive part encryption key from part ID and object key
 	mac.Write(partIDbin[:])
 	partEncryptionKey := mac.Sum(nil)
 
@@ -743,8 +728,9 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 // but has an invalid size.
 func (o ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
-		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
+		return -1, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
+
 	if !o.isMultipart() {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
@@ -757,7 +743,7 @@ func (o ObjectInfo) DecryptedSize() (int64, error) {
 	for _, part := range o.Parts {
 		partSize, err := sio.DecryptedSize(uint64(part.Size))
 		if err != nil {
-			return 0, errObjectTampered
+			return -1, errObjectTampered
 		}
 		size += int64(partSize)
 	}
@@ -1103,7 +1089,7 @@ func (o *ObjectInfo) metadataDecrypter(h http.Header) objectMetaDecryptFn {
 	}
 }
 
-// decryptChecksums will attempt to decode checksums and return it/them if set.
+// decryptPartsChecksums will attempt to decode checksums and return it/them if set.
 // if part > 0, and we have the checksum for the part that will be returned.
 func (o *ObjectInfo) decryptPartsChecksums(h http.Header) {
 	data := o.Checksum
@@ -1113,7 +1099,9 @@ func (o *ObjectInfo) decryptPartsChecksums(h http.Header) {
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {
+			if !errors.Is(err, crypto.ErrSecretKeyMismatch) {
 				encLogIf(GlobalContext, err)
+			}
 			return
 		}
 		data = decrypted

cmd/endpoint-ellipses.go

@@ -311,7 +311,7 @@ func GetAllSets(setDriveCount uint64, args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
+				return nil, config.ErrInvalidErasureEndpoints(nil).Msgf("Input args (%s) has duplicate ellipses", args)
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/erasure-common.go

@@ -19,13 +19,9 @@ package cmd
 
 import (
 	"context"
-	"fmt"
-	"io"
 	"math/rand"
 	"sync"
 	"time"
-
-	"github.com/minio/pkg/v3/sync/errgroup"
 )
 
 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
@@ -87,89 +83,3 @@ func (er erasureObjects) getLocalDisks() (newDisks []StorageAPI) {
 	}
 	return newDisks
 }
-
-// readMultipleFiles Reads raw data from all specified files from all disks.
-func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultipleReq, readQuorum int) ([]ReadMultipleResp, error) {
-	resps := make([]chan ReadMultipleResp, len(disks))
-	for i := range resps {
-		resps[i] = make(chan ReadMultipleResp, len(req.Files))
-	}
-	g := errgroup.WithNErrs(len(disks))
-	// Read files in parallel across disks.
-	for index := range disks {
-		index := index
-		g.Go(func() (err error) {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			return disks[index].ReadMultiple(ctx, req, resps[index])
-		}, index)
-	}
-
-	dataArray := make([]ReadMultipleResp, 0, len(req.Files))
-	// Merge results. They should come in order from each.
-	for _, wantFile := range req.Files {
-		quorum := 0
-		toAdd := ReadMultipleResp{
-			Bucket: req.Bucket,
-			Prefix: req.Prefix,
-			File:   wantFile,
-		}
-		for i := range resps {
-			if disks[i] == nil {
-				continue
-			}
-			select {
-			case <-ctx.Done():
-			case gotFile, ok := <-resps[i]:
-				if !ok {
-					continue
-				}
-				if gotFile.Error != "" || !gotFile.Exists {
-					continue
-				}
-				if gotFile.File != wantFile || gotFile.Bucket != req.Bucket || gotFile.Prefix != req.Prefix {
-					continue
-				}
-				quorum++
-				if toAdd.Modtime.After(gotFile.Modtime) || len(gotFile.Data) < len(toAdd.Data) {
-					// Pick latest, or largest to avoid possible truncated entries.
-					continue
-				}
-				toAdd = gotFile
-			}
-		}
-		if quorum < readQuorum {
-			toAdd.Exists = false
-			toAdd.Error = errErasureReadQuorum.Error()
-			toAdd.Data = nil
-		}
-		dataArray = append(dataArray, toAdd)
-	}
-
-	ignoredErrs := []error{
-		errFileNotFound,
-		errVolumeNotFound,
-		errFileVersionNotFound,
-		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-		io.EOF,              // some times we would read without locks, ignore these errors
-		context.DeadlineExceeded,
-		context.Canceled,
-	}
-	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
-
-	errs := g.Wait()
-	for index, err := range errs {
-		if err == nil {
-			continue
-		}
-		if !IsErr(err, ignoredErrs...) {
-			storageLogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-				disks[index], req.Bucket, req.Prefix, err),
-				disks[index].String())
-		}
-	}
-
-	// Return all the metadata.
-	return dataArray, nil
-}

cmd/erasure-decode.go

@@ -48,14 +48,14 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
 		r2b[i] = i
 	}
 	bufs := make([][]byte, len(readers))
-	// Fill buffers
-	b := globalBytePoolCap.Load().Get()
 	shardSize := int(e.ShardSize())
-	if cap(b) < len(readers)*shardSize {
+	var b []byte
-		// We should always have enough capacity, but older objects may be bigger.
+
-		globalBytePoolCap.Load().Put(b)
+	// We should always have enough capacity, but older objects may be bigger
-		b = nil
+	// we do not need stashbuffer for them.
-	} else {
+	if globalBytePoolCap.Load().WidthCap() >= len(readers)*shardSize {
+		// Fill buffers
+		b = globalBytePoolCap.Load().Get()
 		// Seed the buffers.
 		for i := range bufs {
 			bufs[i] = b[i*shardSize : (i+1)*shardSize]

cmd/erasure-healing-common.go

@@ -289,7 +289,7 @@ func hasPartErr(partErrs []int) bool {
 //   - slice of errors about the state of data files on disk - can have
 //     a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
-	errs []error, latestMeta FileInfo, bucket, object string,
+	errs []error, latestMeta FileInfo, filterByETag bool, bucket, object string,
 	scanMode madmin.HealScanMode,
 ) (availableDisks []StorageAPI, dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
 	availableDisks = make([]StorageAPI, len(onlineDisks))
@@ -345,7 +345,14 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 
 		meta := partsMetadata[i]
-		if !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir {
+		corrupted := false
+		if filterByETag {
+			corrupted = meta.Metadata["etag"] != latestMeta.Metadata["etag"]
+		} else {
+			corrupted = !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir
+		}
+
+		if corrupted {
 			metaErrs[i] = errFileCorrupt
 			partsMetadata[i] = FileInfo{}
 			continue
@@ -408,7 +415,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			verifyResp *CheckPartsResp
 		)
 
-		meta.DataDir = latestMeta.DataDir
 		switch scanMode {
 		case madmin.HealDeepScan:
 			// disk has a valid xl.meta but may not have all the

cmd/erasure-healing-common_test.go

@@ -314,7 +314,7 @@ func TestListOnlineDisks(t *testing.T) {
 					test.expectedTime, modTime)
 			}
 			availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
-				test.errs, fi, bucket, object, madmin.HealDeepScan)
+				test.errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
@@ -496,7 +496,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			}
 
 			availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
-				test.errs, fi, bucket, object, madmin.HealDeepScan)
+				test.errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
@@ -558,7 +558,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)
 
 	filteredDisks, _, dataErrsPerDisk := disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
@@ -580,7 +580,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	errs = make([]error, len(erasureDisks))
 	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
@@ -601,7 +601,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	errs = make([]error, len(erasureDisks))
 	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
@@ -638,7 +638,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	errs = make([]error, len(erasureDisks))
 	filteredDisks, dataErrsPerDisk, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))

cmd/erasure-healing.go

@@ -226,21 +226,29 @@ func (er *erasureObjects) auditHealObject(ctx context.Context, bucket, object, v
 	opts := AuditLogOptions{
 		Event:     "HealObject",
 		Bucket:    bucket,
-		Object:    object,
+		Object:    decodeDirObject(object),
 		VersionID: versionID,
 	}
 	if err != nil {
 		opts.Error = err.Error()
 	}
 
-	opts.Tags = map[string]interface{}{
+	b, a := result.GetCorruptedCounts()
-		"healResult": result,
+	if b > 0 && b == a {
-		"objectLocation": auditObjectOp{
+		opts.Error = fmt.Sprintf("unable to heal %d corrupted blocks on drives", b)
-			Name:   decodeDirObject(object),
+	}
+
+	b, a = result.GetMissingCounts()
+	if b > 0 && b == a {
+		opts.Error = fmt.Sprintf("unable to heal %d missing blocks on drives", b)
+	}
+
+	opts.Tags = map[string]string{
+		"healObject": auditObjectOp{
+			Name: opts.Object,
 			Pool: er.poolIndex + 1,
 			Set:  er.setIndex + 1,
-			Drives: er.getEndpointStrings(),
+		}.String(),
-		},
 	}
 
 	auditLogInternal(ctx, opts)
@@ -320,15 +328,18 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 	// List of disks having latest version of the object xl.meta
 	// (by modtime).
-	onlineDisks, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum)
+	onlineDisks, quorumModTime, quorumETag := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum)
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, etag, readQuorum)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, quorumModTime, quorumETag, readQuorum)
 	if err != nil {
 		return result, err
 	}
 
+	// No modtime quorum
+	filterDisksByETag := quorumETag != ""
+
 	// List of disks having all parts as per latest metadata.
 	// NOTE: do not pass in latestDisks to diskWithAllParts since
 	// the diskWithAllParts needs to reach the drive to ensure
@@ -339,7 +350,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
 	availableDisks, dataErrsByDisk, dataErrsByPart := disksWithAllParts(ctx, onlineDisks, partsMetadata,
-		errs, latestMeta, bucket, object, scanMode)
+		errs, latestMeta, filterDisksByETag, bucket, object, scanMode)
 
 	var erasure Erasure
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
@@ -414,7 +425,14 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	cannotHeal := !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks
+	if cannotHeal && quorumETag != "" {
+		// This is an object that is supposed to be removed by the dangling code
+		// but we noticed that ETag is the same for all objects, let's give it a shot
+		cannotHeal = false
+	}
+
+	if cannotHeal {
 		// Allow for dangling deletes, on versions that have DataDir missing etc.
 		// this would end up restoring the correct readable versions.
 		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrsByPart, ObjectOptions{

cmd/erasure-metadata.go

@@ -216,6 +216,7 @@ func (fi FileInfo) ReplicationInfoEquals(ofi FileInfo) bool {
 }
 
 // objectPartIndex - returns the index of matching object part number.
+// Returns -1 if the part cannot be found.
 func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 	for i, part := range parts {
 		if partNumber == part.Number {
@@ -225,6 +226,17 @@ func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 	return -1
 }
 
+// objectPartIndexNums returns the index of the specified part number.
+// Returns -1 if the part cannot be found.
+func objectPartIndexNums(parts []int, partNumber int) int {
+	for i, part := range parts {
+		if part != 0 && partNumber == part {
+			return i
+		}
+	}
+	return -1
+}
+
 // AddObjectPart - add a new object part in order.
 func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, modTime time.Time, idx []byte, checksums map[string]string) {
 	partInfo := ObjectPartInfo{
@@ -289,13 +301,14 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 		mtimeValid := meta.ModTime.Equal(modTime)
 		if mtimeValid || etagOnly {
 			fmt.Fprintf(h, "%v", meta.XLV1)
-			if !etagOnly {
-				// Verify dataDir is same only when mtime is valid and etag is not considered.
-				fmt.Fprintf(h, "%v", meta.GetDataDir())
-			}
 			for _, part := range meta.Parts {
 				fmt.Fprintf(h, "part.%d", part.Number)
+				fmt.Fprintf(h, "part.%d", part.Size)
 			}
+			// Previously we checked if we had quorum on DataDir value.
+			// We have removed this check to allow reading objects with different DataDir
+			// values in a few drives (due to a rebalance-stop race bug)
+			// provided their their etags or ModTimes match.
 
 			if !meta.Deleted && meta.Size != 0 {
 				fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks)
@@ -390,8 +403,7 @@ func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Tim
 	return findFileInfoInQuorum(ctx, metaArr, modTime, etag, quorum)
 }
 
-// writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
+func writeAllMetadataWithRevert(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int, revert bool) ([]StorageAPI, error) {
-func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	// Start writing `xl.meta` to all disks in parallel.
@@ -415,9 +427,37 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bu
 	mErrs := g.Wait()
 
 	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
+	if err != nil && revert {
+		ng := errgroup.WithNErrs(len(disks))
+		for index := range disks {
+			if mErrs[index] != nil {
+				continue
+			}
+			index := index
+			ng.Go(func() error {
+				if disks[index] == nil {
+					return errDiskNotFound
+				}
+				return disks[index].Delete(ctx, bucket, pathJoin(prefix, xlStorageFormatFile), DeleteOptions{
+					Recursive: true,
+				})
+			}, index)
+		}
+		ng.Wait()
+	}
+
 	return evalDisks(disks, mErrs), err
 }
 
+func writeAllMetadata(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+	return writeAllMetadataWithRevert(ctx, disks, origbucket, bucket, prefix, files, quorum, true)
+}
+
+// writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
+func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+	return writeAllMetadataWithRevert(ctx, disks, origbucket, bucket, prefix, files, quorum, false)
+}
+
 func commonParity(parities []int, defaultParityCount int) int {
 	N := len(parities)
 

cmd/erasure-multipart.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/mimedb"
 	"github.com/minio/pkg/v3/sync/errgroup"
+	"github.com/minio/sio"
 )
 
 func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
@@ -80,6 +81,14 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 		return fi, nil, err
 	}
 
+	if readQuorum < 0 {
+		return fi, nil, errErasureReadQuorum
+	}
+
+	if writeQuorum < 0 {
+		return fi, nil, errErasureWriteQuorum
+	}
+
 	quorum := readQuorum
 	if write {
 		quorum = writeQuorum
@@ -88,14 +97,13 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	// List all online disks.
 	_, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, quorum)
 
-	var reducedErr error
 	if write {
-		reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+		err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	} else {
-		reducedErr = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
+		err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
 	}
-	if reducedErr != nil {
+	if err != nil {
-		return fi, nil, reducedErr
+		return fi, nil, err
 	}
 
 	// Pick one from the first valid metadata.
@@ -103,7 +111,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	return fi, partsMetadata, err
 }
 
-// cleanMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart.
+// cleanupMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart.
 // do not use this function outside of completeMultipartUpload()
 func (er erasureObjects) cleanupMultipartPath(ctx context.Context, paths ...string) {
 	storageDisks := er.getDisks()
@@ -251,7 +259,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 // towards simplification of multipart APIs.
 // The resulting ListMultipartsInfo structure is unmarshalled directly as XML.
 func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
 
 	result.MaxUploads = maxUploads
 	result.KeyMarker = keyMarker
@@ -447,8 +455,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	}
 	fi.DataDir = mustGetUUID()
 
-	if userDefined[ReplicationSsecChecksumHeader] != "" {
+	if ckSum := userDefined[ReplicationSsecChecksumHeader]; ckSum != "" {
-		fi.Checksum, _ = base64.StdEncoding.DecodeString(userDefined[ReplicationSsecChecksumHeader])
+		v, err := base64.StdEncoding.DecodeString(ckSum)
+		if err == nil {
+			fi.Checksum = v
+		}
 		delete(userDefined, ReplicationSsecChecksumHeader)
 	}
 
@@ -490,9 +501,10 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)
 
 	// Write updated `xl.meta` to all disks.
-	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
+	if _, err := writeAllMetadata(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
 		return nil, toObjectErr(err, bucket, object)
 	}
+
 	return &NewMultipartUploadResult{
 		UploadID:     uploadID,
 		ChecksumAlgo: userDefined[hash.MinIOMultipartChecksum],
@@ -506,14 +518,22 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 // Implements S3 compatible initiate multipart API.
 func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "NewMultipartUpload", object, &er)
 	}
 
 	return er.newMultipartUpload(ctx, bucket, object, opts)
 }
 
 // renamePart - renames multipart part to its relevant location under uploadID.
-func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
+func (er erasureObjects) renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, optsMeta []byte, writeQuorum int) ([]StorageAPI, error) {
+	paths := []string{
+		dstEntry,
+		dstEntry + ".meta",
+	}
+
+	// cleanup existing paths first across all drives.
+	er.cleanupMultipartPath(ctx, paths...)
+
 	g := errgroup.WithNErrs(len(disks))
 
 	// Rename file on all underlying storage disks.
@@ -523,60 +543,20 @@ func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, ds
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			return disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry)
+			return disks[index].RenamePart(ctx, srcBucket, srcEntry, dstBucket, dstEntry, optsMeta)
 		}, index)
 	}
 
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
-	// Do not need to undo partial successful operation since those will be cleaned up
-	// in 24hrs via multipart cleaner, never rename() back to `.minio.sys/tmp` as there
-	// is no way to clean them.
-
-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
-	return evalDisks(disks, errs), reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-}
-
-// writeAllDisks - writes 'b' to all provided disks.
-// If write cannot reach quorum, the files will be deleted from all disks.
-func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry string, b []byte, writeQuorum int) ([]StorageAPI, error) {
-	g := errgroup.WithNErrs(len(disks))
-
-	// Write file to all underlying storage disks.
-	for index := range disks {
-		index := index
-		g.Go(func() error {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			return disks[index].WriteAll(ctx, dstBucket, dstEntry, b)
-		}, index)
-	}
-
-	// Wait for all renames to finish.
-	errs := g.Wait()
-
-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if errors.Is(err, errErasureWriteQuorum) {
+	if err != nil {
-		// Remove all written
+		er.cleanupMultipartPath(ctx, paths...)
-		g := errgroup.WithNErrs(len(disks))
-		for index := range disks {
-			if disks[index] == nil || errs[index] != nil {
-				continue
-			}
-			index := index
-			g.Go(func() error {
-				return disks[index].Delete(ctx, dstBucket, dstEntry, DeleteOptions{Immediate: true})
-			}, index)
-		}
-		// Ignore these errors.
-		g.WaitErr()
 	}
 
+	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
+	// otherwise return failure. Cleanup successful renames.
 	return evalDisks(disks, errs), err
 }
 
@@ -587,7 +567,7 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "PutObjectPart", object, &er)
 	}
 
 	data := r.Reader
@@ -597,19 +577,9 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(errInvalidArgument)
 	}
 
-	// Read lock for upload id.
-	// Only held while reading the upload metadata.
-	uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return PartInfo{}, err
-	}
-	rctx := rlkctx.Context()
-	defer uploadIDRLock.RUnlock(rlkctx)
-
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	// Validates if upload ID exists.
-	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
+	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
 			return pi, toObjectErr(err, bucket)
@@ -617,19 +587,6 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	// Write lock for this part ID, only hold it if we are planning to read from the
-	// streamto avoid any concurrent updates.
-	//
-	// Must be held throughout this call.
-	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
-	plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return PartInfo{}, err
-	}
-
-	ctx = plkctx.Context()
-	defer partIDLock.Unlock(plkctx)
-
 	onlineDisks := er.getDisks()
 	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
 
@@ -732,19 +689,6 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	// Rename temporary part file to its final location.
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
-	onlineDisks, err = renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
-	if err != nil {
-		if errors.Is(err, errFileNotFound) {
-			// An in-quorum errFileNotFound means that client stream
-			// prematurely closed and we do not find any xl.meta or
-			// part.1's - in such a scenario we must return as if client
-			// disconnected. This means that erasure.Encode() CreateFile()
-			// did not do anything.
-			return pi, IncompleteBody{Bucket: bucket, Object: object}
-		}
-
-		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
-	}
 
 	md5hex := r.MD5CurrentHexString()
 	if opts.PreserveETag != "" {
@@ -756,25 +700,61 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		index = opts.IndexCB()
 	}
 
+	actualSize := data.ActualSize()
+	if actualSize < 0 {
+		_, encrypted := crypto.IsEncrypted(fi.Metadata)
+		compressed := fi.IsCompressed()
+		switch {
+		case compressed:
+			// ... nothing changes for compressed stream.
+			// if actualSize is -1 we have no known way to
+			// determine what is the actualSize.
+		case encrypted:
+			decSize, err := sio.DecryptedSize(uint64(n))
+			if err == nil {
+				actualSize = int64(decSize)
+			}
+		default:
+			actualSize = n
+		}
+	}
+
 	partInfo := ObjectPartInfo{
 		Number:     partID,
 		ETag:       md5hex,
 		Size:       n,
-		ActualSize: data.ActualSize(),
+		ActualSize: actualSize,
 		ModTime:    UTCNow(),
 		Index:      index,
 		Checksums:  r.ContentCRC(),
 	}
 
-	fi.Parts = []ObjectPartInfo{partInfo}
+	partFI, err := partInfo.MarshalMsg(nil)
-	partFI, err := fi.MarshalMsg(nil)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
-	// Write part metadata to all disks.
+	// Serialize concurrent part uploads.
-	onlineDisks, err = writeAllDisks(ctx, onlineDisks, minioMetaMultipartBucket, partPath+".meta", partFI, writeQuorum)
+	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
+	plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
+		return PartInfo{}, err
+	}
+
+	ctx = plkctx.Context()
+	defer partIDLock.Unlock(plkctx)
+
+	onlineDisks, err = er.renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, partFI, writeQuorum)
+	if err != nil {
+		if errors.Is(err, errFileNotFound) {
+			// An in-quorum errFileNotFound means that client stream
+			// prematurely closed and we do not find any xl.meta or
+			// part.1's - in such a scenario we must return as if client
+			// disconnected. This means that erasure.Encode() CreateFile()
+			// did not do anything.
+			return pi, IncompleteBody{Bucket: bucket, Object: object}
+		}
+
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
@@ -799,7 +779,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 // Does not contain currently uploaded parts by design.
 func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "GetMultipartInfo", object, &er)
 	}
 
 	result := MultipartInfo{
@@ -808,14 +788,6 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
-	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return MultipartInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer uploadIDLock.RUnlock(lkctx)
-
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
@@ -828,6 +800,61 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 	return result, nil
 }
 
+func (er erasureObjects) listParts(ctx context.Context, onlineDisks []StorageAPI, partPath string, readQuorum int) ([]int, error) {
+	g := errgroup.WithNErrs(len(onlineDisks))
+
+	objectParts := make([][]string, len(onlineDisks))
+	// List uploaded parts from drives.
+	for index := range onlineDisks {
+		index := index
+		g.Go(func() (err error) {
+			if onlineDisks[index] == nil {
+				return errDiskNotFound
+			}
+			objectParts[index], err = onlineDisks[index].ListDir(ctx, minioMetaMultipartBucket, minioMetaMultipartBucket, partPath, -1)
+			return err
+		}, index)
+	}
+
+	if err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum); err != nil {
+		return nil, err
+	}
+
+	partQuorumMap := make(map[int]int)
+	for _, driveParts := range objectParts {
+		partsWithMetaCount := make(map[int]int, len(driveParts))
+		// part files can be either part.N or part.N.meta
+		for _, partPath := range driveParts {
+			var partNum int
+			if _, err := fmt.Sscanf(partPath, "part.%d", &partNum); err == nil {
+				partsWithMetaCount[partNum]++
+				continue
+			}
+			if _, err := fmt.Sscanf(partPath, "part.%d.meta", &partNum); err == nil {
+				partsWithMetaCount[partNum]++
+			}
+		}
+		// Include only part.N.meta files with corresponding part.N
+		for partNum, cnt := range partsWithMetaCount {
+			if cnt < 2 {
+				continue
+			}
+			partQuorumMap[partNum]++
+		}
+	}
+
+	var partNums []int
+	for partNum, count := range partQuorumMap {
+		if count < readQuorum {
+			continue
+		}
+		partNums = append(partNums, partNum)
+	}
+
+	sort.Ints(partNums)
+	return partNums, nil
+}
+
 // ListObjectParts - lists all previously uploaded parts for a given
 // object and uploadID.  Takes additional input of part-number-marker
 // to indicate where the listing should begin from.
@@ -837,23 +864,23 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "ListObjectParts", object, &er)
 	}
 
-	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return ListPartsInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer uploadIDLock.RUnlock(lkctx)
-
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+	if partNumberMarker < 0 {
+		partNumberMarker = 0
+	}
+
+	// Limit output to maxPartsList.
+	if maxParts > maxPartsList {
+		maxParts = maxPartsList
+	}
 
 	// Populate the result stub.
 	result.Bucket = bucket
@@ -864,128 +891,171 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.UserDefined = cloneMSS(fi.Metadata)
 	result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
 
-	if partNumberMarker < 0 {
-		partNumberMarker = 0
-	}
-
-	// Limit output to maxPartsList.
-	if maxParts > maxPartsList-partNumberMarker {
-		maxParts = maxPartsList - partNumberMarker
-	}
-
 	if maxParts == 0 {
 		return result, nil
 	}
 
+	onlineDisks := er.getDisks()
+	readQuorum := fi.ReadQuorum(er.defaultRQuorum())
 	// Read Part info for all parts
-	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
+	partPath := pathJoin(uploadIDPath, fi.DataDir) + SlashSeparator
-	req := ReadMultipleReq{
-		Bucket:       minioMetaMultipartBucket,
-		Prefix:       partPath,
-		MaxSize:      1 << 20, // Each part should realistically not be > 1MiB.
-		MaxResults:   maxParts + 1,
-		MetadataOnly: true,
-	}
 
-	start := partNumberMarker + 1
+	// List parts in quorum
-	end := start + maxParts
+	partNums, err := er.listParts(ctx, onlineDisks, partPath, readQuorum)
-
-	// Parts are 1 based, so index 0 is part one, etc.
-	for i := start; i <= end; i++ {
-		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
-	}
-
-	var disk StorageAPI
-	disks := er.getOnlineLocalDisks()
-	if len(disks) == 0 {
-		// using er.getOnlineLocalDisks() has one side-affect where
-		// on a pooled setup all disks are remote, add a fallback
-		disks = er.getOnlineDisks()
-	}
-
-	for _, disk = range disks {
-		if disk == nil {
-			continue
-		}
-
-		if !disk.IsOnline() {
-			continue
-		}
-
-		break
-	}
-
-	g := errgroup.WithNErrs(len(req.Files)).WithConcurrency(32)
-
-	partsInfo := make([]ObjectPartInfo, len(req.Files))
-	for i, file := range req.Files {
-		file := file
-		partN := i + start
-		i := i
-
-		g.Go(func() error {
-			buf, err := disk.ReadAll(ctx, minioMetaMultipartBucket, pathJoin(partPath, file))
 	if err != nil {
-				return err
+		// This means that fi.DataDir, is not yet populated so we
-			}
+		// return an empty response.
-
+		if errors.Is(err, errFileNotFound) {
-			var pfi FileInfo
-			_, err = pfi.UnmarshalMsg(buf)
-			if err != nil {
-				return err
-			}
-
-			if len(pfi.Parts) != 1 {
-				return errors.New("invalid number of parts expected 1, got 0")
-			}
-
-			if partN != pfi.Parts[0].Number {
-				return fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partN, partN, pfi.Parts[0].Number)
-			}
-
-			partsInfo[i] = pfi.Parts[0]
-			return nil
-		}, i)
-	}
-
-	g.Wait()
-
-	for _, part := range partsInfo {
-		if part.Number != 0 && !part.ModTime.IsZero() {
-			fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums)
-		}
-	}
-
-	// Only parts with higher part numbers will be listed.
-	parts := fi.Parts
-	result.Parts = make([]PartInfo, 0, len(parts))
-	for _, part := range parts {
-		result.Parts = append(result.Parts, PartInfo{
-			PartNumber:     part.Number,
-			ETag:           part.ETag,
-			LastModified:   part.ModTime,
-			ActualSize:     part.ActualSize,
-			Size:           part.Size,
-			ChecksumCRC32:  part.Checksums["CRC32"],
-			ChecksumCRC32C: part.Checksums["CRC32C"],
-			ChecksumSHA1:   part.Checksums["SHA1"],
-			ChecksumSHA256: part.Checksums["SHA256"],
-		})
-		if len(result.Parts) >= maxParts {
-			break
-		}
-	}
-
-	// If listed entries are more than maxParts, we set IsTruncated as true.
-	if len(parts) > len(result.Parts) {
-		result.IsTruncated = true
-		// Make sure to fill next part number marker if IsTruncated is
-		// true for subsequent listing.
-		nextPartNumberMarker := result.Parts[len(result.Parts)-1].PartNumber
-		result.NextPartNumberMarker = nextPartNumberMarker
-	}
 			return result, nil
 		}
+		return result, toObjectErr(err, bucket, object, uploadID)
+	}
+
+	if len(partNums) == 0 {
+		return result, nil
+	}
+
+	start := objectPartIndexNums(partNums, partNumberMarker)
+	if start != -1 {
+		partNums = partNums[start+1:]
+	}
+
+	result.Parts = make([]PartInfo, 0, len(partNums))
+	partMetaPaths := make([]string, len(partNums))
+	for i, part := range partNums {
+		partMetaPaths[i] = pathJoin(partPath, fmt.Sprintf("part.%d.meta", part))
+	}
+
+	// Read parts in quorum
+	objParts, err := readParts(ctx, onlineDisks, minioMetaMultipartBucket, partMetaPaths,
+		partNums, readQuorum)
+	if err != nil {
+		return result, toObjectErr(err, bucket, object, uploadID)
+	}
+
+	count := maxParts
+	for _, objPart := range objParts {
+		result.Parts = append(result.Parts, PartInfo{
+			PartNumber:     objPart.Number,
+			LastModified:   objPart.ModTime,
+			ETag:           objPart.ETag,
+			Size:           objPart.Size,
+			ActualSize:     objPart.ActualSize,
+			ChecksumCRC32:  objPart.Checksums["CRC32"],
+			ChecksumCRC32C: objPart.Checksums["CRC32C"],
+			ChecksumSHA1:   objPart.Checksums["SHA1"],
+			ChecksumSHA256: objPart.Checksums["SHA256"],
+		})
+		count--
+		if count == 0 {
+			break
+		}
+	}
+
+	if len(objParts) > len(result.Parts) {
+		result.IsTruncated = true
+		// Make sure to fill next part number marker if IsTruncated is true for subsequent listing.
+		result.NextPartNumberMarker = result.Parts[len(result.Parts)-1].PartNumber
+	}
+
+	return result, nil
+}
+
+func readParts(ctx context.Context, disks []StorageAPI, bucket string, partMetaPaths []string, partNumbers []int, readQuorum int) ([]ObjectPartInfo, error) {
+	g := errgroup.WithNErrs(len(disks))
+
+	objectPartInfos := make([][]*ObjectPartInfo, len(disks))
+	// Rename file on all underlying storage disks.
+	for index := range disks {
+		index := index
+		g.Go(func() (err error) {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			objectPartInfos[index], err = disks[index].ReadParts(ctx, bucket, partMetaPaths...)
+			return err
+		}, index)
+	}
+
+	if err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum); err != nil {
+		return nil, err
+	}
+
+	partInfosInQuorum := make([]ObjectPartInfo, len(partMetaPaths))
+	for pidx := range partMetaPaths {
+		// partMetaQuorumMap uses
+		//  - path/to/part.N as key to collate errors from failed drives.
+		//  - part ETag to collate part metadata
+		partMetaQuorumMap := make(map[string]int, len(partNumbers))
+		var pinfos []*ObjectPartInfo
+		for idx := range disks {
+			if len(objectPartInfos[idx]) != len(partMetaPaths) {
+				partMetaQuorumMap[partMetaPaths[pidx]]++
+				continue
+			}
+
+			pinfo := objectPartInfos[idx][pidx]
+			if pinfo != nil && pinfo.ETag != "" {
+				pinfos = append(pinfos, pinfo)
+				partMetaQuorumMap[pinfo.ETag]++
+				continue
+			}
+			partMetaQuorumMap[partMetaPaths[pidx]]++
+		}
+
+		var maxQuorum int
+		var maxETag string
+		var maxPartMeta string
+		for etag, quorum := range partMetaQuorumMap {
+			if maxQuorum < quorum {
+				maxQuorum = quorum
+				maxETag = etag
+				maxPartMeta = etag
+			}
+		}
+		// found is a representative ObjectPartInfo which either has the maximally occurring ETag or an error.
+		var found *ObjectPartInfo
+		for _, pinfo := range pinfos {
+			if pinfo == nil {
+				continue
+			}
+			if maxETag != "" && pinfo.ETag == maxETag {
+				found = pinfo
+				break
+			}
+			if pinfo.ETag == "" && maxPartMeta != "" && path.Base(maxPartMeta) == fmt.Sprintf("part.%d.meta", pinfo.Number) {
+				found = pinfo
+				break
+			}
+		}
+
+		if found != nil && found.ETag != "" && partMetaQuorumMap[maxETag] >= readQuorum {
+			partInfosInQuorum[pidx] = *found
+			continue
+		}
+		partInfosInQuorum[pidx] = ObjectPartInfo{
+			Number: partNumbers[pidx],
+			Error: InvalidPart{
+				PartNumber: partNumbers[pidx],
+			}.Error(),
+		}
+
+	}
+	return partInfosInQuorum, nil
+}
+
+func objPartToPartErr(part ObjectPartInfo) error {
+	if strings.Contains(part.Error, "file not found") {
+		return InvalidPart{PartNumber: part.Number}
+	}
+	if strings.Contains(part.Error, "Specified part could not be found") {
+		return InvalidPart{PartNumber: part.Number}
+	}
+	if strings.Contains(part.Error, errErasureReadQuorum.Error()) {
+		return errErasureReadQuorum
+	}
+	return errors.New(part.Error)
+}
 
 // CompleteMultipartUpload - completes an ongoing multipart
 // transaction after receiving all the parts indicated by the client.
@@ -995,7 +1065,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 // Implements S3 compatible Complete multipart API.
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er)
 	}
 
 	if opts.CheckPrecondFn != nil {
@@ -1019,16 +1089,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
-	// Hold write locks to verify uploaded parts, also disallows any
-	// parallel PutObjectPart() requests.
-	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return oi, err
-	}
-	ctx = wlkctx.Context()
-	defer uploadIDLock.Unlock(wlkctx)
-
 	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
@@ -1040,24 +1100,22 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	onlineDisks := er.getDisks()
 	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
+	readQuorum := fi.ReadQuorum(er.defaultRQuorum())
 
 	// Read Part info for all parts
-	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
+	partPath := pathJoin(uploadIDPath, fi.DataDir) + SlashSeparator
-	req := ReadMultipleReq{
+	partMetaPaths := make([]string, len(parts))
-		Bucket:       minioMetaMultipartBucket,
+	partNumbers := make([]int, len(parts))
-		Prefix:       partPath,
+	for idx, part := range parts {
-		MaxSize:      1 << 20, // Each part should realistically not be > 1MiB.
+		partMetaPaths[idx] = pathJoin(partPath, fmt.Sprintf("part.%d.meta", part.PartNumber))
-		Files:        make([]string, 0, len(parts)),
+		partNumbers[idx] = part.PartNumber
-		AbortOn404:   true,
-		MetadataOnly: true,
 	}
-	for _, part := range parts {
+
-		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", part.PartNumber))
+	partInfoFiles, err := readParts(ctx, onlineDisks, minioMetaMultipartBucket, partMetaPaths, partNumbers, readQuorum)
-	}
-	partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
 	if err != nil {
 		return oi, err
 	}
+
 	if len(partInfoFiles) != len(parts) {
 		// Should only happen through internal error
 		err := fmt.Errorf("unexpected part result count: %d, want %d", len(partInfoFiles), len(parts))
@@ -1083,7 +1141,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	// However, in case of encryption, the persisted part ETags don't match
 	// what we have sent to the client during PutObjectPart. The reason is
 	// that ETags are encrypted. Hence, the client will send a list of complete
-	// part ETags of which non can match the ETag of any part. For example
+	// part ETags of which may not match the ETag of any part. For example
 	//   ETag (client):          30902184f4e62dd8f98f0aaff810c626
 	//   ETag (server-internal): 20000f00ce5dc16e3f3b124f586ae1d88e9caa1c598415c2759bbb50e84a59f630902184f4e62dd8f98f0aaff810c626
 	//
@@ -1119,35 +1177,22 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		opts.EncryptFn = metadataEncrypter(key)
 	}
 
-	for i, part := range partInfoFiles {
+	for idx, part := range partInfoFiles {
-		partID := parts[i].PartNumber
+		if part.Error != "" {
-		if part.Error != "" || !part.Exists {
+			err = objPartToPartErr(part)
-			return oi, InvalidPart{
-				PartNumber: partID,
-			}
-		}
-
-		var pfi FileInfo
-		_, err := pfi.UnmarshalMsg(part.Data)
-		if err != nil {
-			// Maybe crash or similar.
 			bugLogIf(ctx, err)
-			return oi, InvalidPart{
+			return oi, err
-				PartNumber: partID,
-			}
 		}
 
-		partI := pfi.Parts[0]
+		if parts[idx].PartNumber != part.Number {
-		partNumber := partI.Number
+			internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", parts[idx].PartNumber, parts[idx].PartNumber, part.Number))
-		if partID != partNumber {
-			internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
 			return oi, InvalidPart{
-				PartNumber: partID,
+				PartNumber: part.Number,
 			}
 		}
 
 		// Add the current part.
-		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index, partI.Checksums)
+		fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums)
 	}
 
 	// Calculate full object size.
@@ -1253,16 +1298,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
-	if !opts.NoLock {
-		lk := er.NewNSLock(bucket, object)
-		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-		if err != nil {
-			return ObjectInfo{}, err
-		}
-		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx)
-	}
-
 	// Accept encrypted checksum from incoming request.
 	if opts.UserDefined[ReplicationSsecChecksumHeader] != "" {
 		if v, err := base64.StdEncoding.DecodeString(opts.UserDefined[ReplicationSsecChecksumHeader]); err == nil {
@@ -1301,7 +1336,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	// Save the consolidated actual size.
 	if opts.ReplicationRequest {
-		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = opts.UserDefined["X-Minio-Internal-Actual-Object-Size"]
+		if v := opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"]; v != "" {
+			fi.Metadata[ReservedMetadataPrefix+"actual-size"] = v
+		}
 	} else {
 		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 	}
@@ -1339,6 +1376,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	if !opts.NoLock {
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
 
 	defer func() {
@@ -1359,13 +1406,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	if !opts.Speedtest && len(versions) > 0 {
-		globalMRFState.addPartialOp(partialOperation{
+		globalMRFState.addPartialOp(PartialOperation{
-			bucket:    bucket,
+			Bucket:    bucket,
-			object:    object,
+			Object:    object,
-			queued:    time.Now(),
+			Queued:    time.Now(),
-			versions:  versions,
+			Versions:  versions,
-			setIndex:  er.setIndex,
+			SetIndex:  er.setIndex,
-			poolIndex: er.poolIndex,
+			PoolIndex: er.poolIndex,
 		})
 	}
 
@@ -1405,17 +1452,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
 	}
 
-	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx)
-
 	// Validates if upload ID exists.
 	if _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false); err != nil {
 		if errors.Is(err, errVolumeNotFound) {

cmd/erasure-object.go

@@ -32,7 +32,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/dustin/go-humanize"
 	"github.com/klauspost/readahead"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -49,6 +48,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/mimedb"
 	"github.com/minio/pkg/v3/sync/errgroup"
+	"github.com/minio/sio"
 )
 
 // list all errors which can be ignored in object operations.
@@ -70,7 +70,7 @@ func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
 // update metadata.
 func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
 	if !dstOpts.NoAuditLog {
-		auditObjectErasureSet(ctx, dstObject, &er)
+		auditObjectErasureSet(ctx, "CopyObject", dstObject, &er)
 	}
 
 	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
@@ -97,12 +97,12 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	if srcOpts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, storageDisks, "", srcBucket, srcObject, srcOpts.VersionID, true, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false, true)
+		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false)
 	}
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(srcBucket, minioMetaBucket) {
+		if shouldCheckForDangling(err, errs, srcBucket) {
 			_, derr := er.deleteIfDangling(context.Background(), srcBucket, srcObject, metaArr, errs, nil, srcOpts)
 			if derr == nil {
 				if srcOpts.VersionID != "" {
@@ -199,7 +199,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "GetObject", object, &er)
 	}
 
 	var unlockOnDefer bool
@@ -395,24 +395,16 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			// that we have some parts or data blocks missing or corrupted
 			// - attempt a heal to successfully heal them for future calls.
 			if written == partLength {
-				var scan madmin.HealScanMode
+				if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) {
-				switch {
-				case errors.Is(err, errFileNotFound):
-					scan = madmin.HealNormalScan
-				case errors.Is(err, errFileCorrupt):
-					scan = madmin.HealDeepScan
-				}
-				switch scan {
-				case madmin.HealNormalScan, madmin.HealDeepScan:
 					healOnce.Do(func() {
-						globalMRFState.addPartialOp(partialOperation{
+						globalMRFState.addPartialOp(PartialOperation{
-							bucket:    bucket,
+							Bucket:     bucket,
-							object:    object,
+							Object:     object,
-							versionID: fi.VersionID,
+							VersionID:  fi.VersionID,
-							queued:    time.Now(),
+							Queued:     time.Now(),
-							setIndex:  er.setIndex,
+							SetIndex:   er.setIndex,
-							poolIndex: er.poolIndex,
+							PoolIndex:  er.poolIndex,
-							scanMode:  scan,
+							BitrotScan: errors.Is(err, errFileCorrupt),
 						})
 					})
 					// Healing is triggered and we have written
@@ -439,7 +431,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "GetObjectInfo", object, &er)
 	}
 
 	if !opts.NoLock {
@@ -456,7 +448,7 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
 
-func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]interface{}) {
+func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]string) {
 	if len(logger.AuditTargets()) == 0 {
 		return
 	}
@@ -472,13 +464,16 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
 	auditLogInternal(ctx, opts)
 }
 
-func joinErrs(errs []error) []string {
+func joinErrs(errs []error) string {
-	s := make([]string, len(errs))
+	var s string
 	for i := range s {
+		if s != "" {
+			s += ","
+		}
 		if errs[i] == nil {
-			s[i] = "<nil>"
+			s += "<nil>"
 		} else {
-			s[i] = errs[i].Error()
+			s += errs[i].Error()
 		}
 	}
 	return s
@@ -491,20 +486,18 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		// can be deleted safely, in such a scenario return ReadQuorum error.
 		return FileInfo{}, errErasureReadQuorum
 	}
-	tags := make(map[string]interface{}, 4)
+	tags := make(map[string]string, 16)
-	tags["set"] = er.setIndex
+	tags["set"] = strconv.Itoa(er.setIndex)
-	tags["pool"] = er.poolIndex
+	tags["pool"] = strconv.Itoa(er.poolIndex)
 	tags["merrs"] = joinErrs(errs)
-	tags["derrs"] = dataErrsByPart
+	tags["derrs"] = fmt.Sprintf("%v", dataErrsByPart)
 	if m.IsValid() {
-		tags["size"] = m.Size
+		tags["sz"] = strconv.FormatInt(m.Size, 10)
-		tags["mtime"] = m.ModTime.Format(http.TimeFormat)
+		tags["mt"] = m.ModTime.Format(iso8601Format)
-		tags["data"] = m.Erasure.DataBlocks
+		tags["d:p"] = fmt.Sprintf("%d:%d", m.Erasure.DataBlocks, m.Erasure.ParityBlocks)
-		tags["parity"] = m.Erasure.ParityBlocks
 	} else {
-		tags["invalid-meta"] = true
+		tags["invalid"] = "1"
-		tags["data"] = er.setDriveCount - er.defaultParityCount
+		tags["d:p"] = fmt.Sprintf("%d:%d", er.setDriveCount-er.defaultParityCount, er.defaultParityCount)
-		tags["parity"] = er.defaultParityCount
 	}
 
 	// count the number of offline disks
@@ -527,7 +520,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		}
 	}
 	if offline > 0 {
-		tags["offline"] = offline
+		tags["offline"] = strconv.Itoa(offline)
 	}
 
 	_, file, line, cok := runtime.Caller(1)
@@ -556,30 +549,23 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		}, index)
 	}
 
-	rmDisks := make(map[string]string, len(disks))
 	for index, err := range g.Wait() {
-		var errStr, diskName string
+		var errStr string
 		if err != nil {
 			errStr = err.Error()
 		} else {
 			errStr = "<nil>"
 		}
-		if disks[index] != nil {
+		tags[fmt.Sprintf("ddisk-%d", index)] = errStr
-			diskName = disks[index].String()
-		} else {
-			diskName = fmt.Sprintf("disk-%d", index)
 	}
-		rmDisks[diskName] = errStr
+
-	}
-	tags["cleanupResult"] = rmDisks
 	return m, nil
 }
 
-func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers, allParts bool) (FileInfo, error) {
+func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers bool) (FileInfo, error) {
 	return getFileInfo(ri.Buf, bucket, object, "", fileInfoOpts{
 		Data:             readData,
 		InclFreeVersions: inclFreeVers,
-		AllParts:         allParts,
 	})
 }
 
@@ -604,7 +590,7 @@ func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object
 	return rawFileInfos, g.Wait()
 }
 
-func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers, allParts bool) ([]FileInfo, []error) {
+func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
 	metadataArray := make([]*xlMetaV2, len(rawFileInfos))
 	metaFileInfos := make([]FileInfo, len(rawFileInfos))
 	metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(rawFileInfos))
@@ -641,7 +627,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 
 	readQuorum := (len(rawFileInfos) + 1) / 2
 	meta := &xlMetaV2{versions: mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)}
-	lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers, allParts)
+	lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers, true)
 	if err != nil {
 		for i := range errs {
 			if errs[i] == nil {
@@ -670,7 +656,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 		}
 
 		// make sure to preserve this for diskmtime based healing bugfix.
-		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, allParts)
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, true)
 		if errs[index] != nil {
 			continue
 		}
@@ -704,20 +690,16 @@ func shouldCheckForDangling(err error, errs []error, bucket string) bool {
 	// Check if we have a read quorum issue
 	case errors.Is(err, errErasureReadQuorum):
 		return true
-	// Check if the object is inexistent in most disks but not all of them
+	// Check if the object is non-existent on most disks but not all of them
-	case errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound):
+	case (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) && (countErrs(errs, nil) > 0):
-		for i := range errs {
-			if errs[i] == nil {
 		return true
 	}
-		}
-	}
 	return false
 }
 
-func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData, inclFreeVers, allParts bool) ([]FileInfo, []error) {
+func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
 	rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, readData)
-	return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers, allParts)
+	return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers)
 }
 
 func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) {
@@ -774,7 +756,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 					// Read the latest version
 					rfi, err = disk.ReadXL(ctx, bucket, object, readData)
 					if err == nil {
-						fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions, true)
+						fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions)
 					}
 				}
 
@@ -820,13 +802,13 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		// additionally do not heal delete markers inline, let them be
 		// healed upon regular heal process.
 		if missingBlocks > 0 && missingBlocks < fi.Erasure.DataBlocks {
-			globalMRFState.addPartialOp(partialOperation{
+			globalMRFState.addPartialOp(PartialOperation{
-				bucket:    fi.Volume,
+				Bucket:    fi.Volume,
-				object:    fi.Name,
+				Object:    fi.Name,
-				versionID: fi.VersionID,
+				VersionID: fi.VersionID,
-				queued:    time.Now(),
+				Queued:    time.Now(),
-				setIndex:  er.setIndex,
+				SetIndex:  er.setIndex,
-				poolIndex: er.poolIndex,
+				PoolIndex: er.poolIndex,
 			})
 		}
 
@@ -912,7 +894,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		// we must use rawFileInfo to resolve versions to figure out the latest version.
 		if opts.VersionID == "" && totalResp == er.setDriveCount {
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
-				rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true))
+				rawArr, errs, bucket, object, readData, opts.InclFreeVersions))
 		} else {
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
 		}
@@ -935,6 +917,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 				}
 			}
 		}
+		// when we have insufficient read quorum and inconsistent metadata return
+		// file not found, since we can't possibly have a way to recover this object
+		// anyway.
+		if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta {
+			if opts.VersionID != "" {
+				err = errFileVersionNotFound
+			} else {
+				err = errFileNotFound
+			}
+		}
 		return fi, nil, nil, toObjectErr(err, bucket, object)
 	}
 
@@ -1252,7 +1244,7 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "PutObject", object, &er)
 	}
 
 	data := r.Reader
@@ -1343,16 +1335,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.DataDir = mustGetUUID()
-	fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
+	if ckSum := userDefined[ReplicationSsecChecksumHeader]; ckSum != "" {
-	if opts.EncryptFn != nil {
+		if v, err := base64.StdEncoding.DecodeString(ckSum); err == nil {
-		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
-	}
-	if userDefined[ReplicationSsecChecksumHeader] != "" {
-		if v, err := base64.StdEncoding.DecodeString(userDefined[ReplicationSsecChecksumHeader]); err == nil {
 			fi.Checksum = v
 		}
-	}
 		delete(userDefined, ReplicationSsecChecksumHeader)
+	}
 	uniqueID := mustGetUUID()
 	tempObj := uniqueID
 
@@ -1392,30 +1380,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 
-	shardFileSize := erasure.ShardFileSize(data.Size())
+	var inlineBuffers []*bytes.Buffer
-	inlineBlock := globalStorageClass.InlineBlock()
+	if globalStorageClass.ShouldInline(erasure.ShardFileSize(data.ActualSize()), opts.Versioned) {
-	if inlineBlock <= 0 {
+		inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-		inlineBlock = 128 * humanize.KiByte
 	}
 
+	shardFileSize := erasure.ShardFileSize(data.Size())
 	writers := make([]io.Writer, len(onlineDisks))
-	var inlineBuffers []*bytes.Buffer
-	if shardFileSize >= 0 {
-		if !opts.Versioned && shardFileSize <= inlineBlock {
-			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-		} else if shardFileSize < inlineBlock/8 {
-			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-		}
-	} else {
-		// If compressed, use actual size to determine.
-		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
-			if !opts.Versioned && sz <= inlineBlock {
-				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-			} else if sz < inlineBlock/8 {
-				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-			}
-		}
-	}
 	for i, disk := range onlineDisks {
 		if disk == nil {
 			continue
@@ -1467,21 +1438,37 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	if opts.IndexCB != nil {
 		compIndex = opts.IndexCB()
 	}
-	if !opts.NoLock {
-		lk := er.NewNSLock(bucket, object)
-		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-		if err != nil {
-			return ObjectInfo{}, err
-		}
-		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx)
-	}
 
 	modTime := opts.MTime
 	if opts.MTime.IsZero() {
 		modTime = UTCNow()
 	}
 
+	kind, encrypted := crypto.IsEncrypted(userDefined)
+	actualSize := data.ActualSize()
+	if actualSize < 0 {
+		compressed := fi.IsCompressed()
+		switch {
+		case compressed:
+			// ... nothing changes for compressed stream.
+			// if actualSize is -1 we have no known way to
+			// determine what is the actualSize.
+		case encrypted:
+			decSize, err := sio.DecryptedSize(uint64(n))
+			if err == nil {
+				actualSize = int64(decSize)
+			}
+		default:
+			actualSize = n
+		}
+	}
+	if fi.Checksum == nil {
+		// Trailing headers checksums should now be filled.
+		fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
+		if opts.EncryptFn != nil {
+			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
+		}
+	}
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -1493,12 +1480,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			partsMetadata[i].Data = nil
 		}
 		// No need to add checksum to part. We already have it on the object.
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, compIndex, nil)
+		partsMetadata[i].AddObjectPart(1, "", n, actualSize, modTime, compIndex, nil)
 		partsMetadata[i].Versioned = opts.Versioned || opts.VersionSuspended
+		partsMetadata[i].Checksum = fi.Checksum
 	}
 
 	userDefined["etag"] = r.MD5CurrentHexString()
-	kind, _ := crypto.IsEncrypted(userDefined)
 	if opts.PreserveETag != "" {
 		if !opts.ReplicationRequest {
 			userDefined["etag"] = opts.PreserveETag
@@ -1534,6 +1521,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 	}
 
+	if !opts.NoLock {
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	// Rename the successfully written temporary object to final location.
 	onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
 	if err != nil {
@@ -1578,13 +1575,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 				break
 			}
 		} else {
-			globalMRFState.addPartialOp(partialOperation{
+			globalMRFState.addPartialOp(PartialOperation{
-				bucket:    bucket,
+				Bucket:    bucket,
-				object:    object,
+				Object:    object,
-				queued:    time.Now(),
+				Queued:    time.Now(),
-				versions:  versions,
+				Versions:  versions,
-				setIndex:  er.setIndex,
+				SetIndex:  er.setIndex,
-				poolIndex: er.poolIndex,
+				PoolIndex: er.poolIndex,
 			})
 		}
 	}
@@ -1627,7 +1624,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
 	if !opts.NoAuditLog {
 		for _, obj := range objects {
-			auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+			auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectV.ObjectName, &er)
 		}
 	}
 
@@ -1847,7 +1844,7 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "DeleteObject", object, &er)
 	}
 
 	var lc *lifecycle.Lifecycle
@@ -2113,11 +2110,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 // Send the successful but partial upload/delete, however ignore
 // if the channel is blocked by other items.
 func (er erasureObjects) addPartial(bucket, object, versionID string) {
-	globalMRFState.addPartialOp(partialOperation{
+	globalMRFState.addPartialOp(PartialOperation{
-		bucket:    bucket,
+		Bucket:    bucket,
-		object:    object,
+		Object:    object,
-		versionID: versionID,
+		VersionID: versionID,
-		queued:    time.Now(),
+		Queued:    time.Now(),
 	})
 }
 
@@ -2142,12 +2139,12 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+		if shouldCheckForDangling(err, errs, bucket) {
 			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr == nil {
 				if opts.VersionID != "" {
@@ -2201,6 +2198,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 // PutObjectTags - replace or add tags to an existing object
 func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) {
+	if !opts.NoLock {
 		// Lock the object before updating tags.
 		lk := er.NewNSLock(bucket, object)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -2209,6 +2207,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 		}
 		ctx = lkctx.Context()
 		defer lk.Unlock(lkctx)
+	}
 
 	disks := er.getDisks()
 
@@ -2219,12 +2218,12 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+		if shouldCheckForDangling(err, errs, bucket) {
 			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr == nil {
 				if opts.VersionID != "" {
@@ -2319,6 +2318,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		return err
 	}
 
+	if !opts.NoLock {
 		// Acquire write lock before starting to transition the object.
 		lk := er.NewNSLock(bucket, object)
 		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
@@ -2327,6 +2327,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		}
 		ctx = lkctx.Context()
 		defer lk.Unlock(lkctx)
+	}
 
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
@@ -2364,6 +2365,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 
 	destObj, err := genTransitionObjName(bucket)
 	if err != nil {
+		traceFn(ILMTransition, nil, err)
 		return err
 	}
 
@@ -2377,6 +2379,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	rv, err = tgtClient.Put(ctx, destObj, pr, fi.Size)
 	pr.CloseWithError(err)
 	if err != nil {
+		traceFn(ILMTransition, nil, err)
 		return err
 	}
 	fi.TransitionStatus = lifecycle.TransitionComplete

cmd/erasure-server-pool-decom.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -490,13 +490,10 @@ const (
 // in 'pool.bin', this is eventually used for decommissioning the pool.
 func (z *erasureServerPools) Init(ctx context.Context) error {
 	// Load rebalance metadata if present
-	err := z.loadRebalanceMeta(ctx)
+	if err := z.loadRebalanceMeta(ctx); err == nil {
-	if err != nil {
+		// Start rebalance routine if we can reload rebalance metadata.
-		return fmt.Errorf("failed to load rebalance data: %w", err)
-	}
-
-	// Start rebalance routine
 		z.StartRebalance()
+	}
 
 	meta := poolMeta{}
 	if err := meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
@@ -608,7 +605,7 @@ func (z *erasureServerPools) IsDecommissionRunning() bool {
 	return false
 }
 
-func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bucket string, gr *GetObjectReader) (err error) {
 	objInfo := gr.ObjInfo
 
 	defer func() {
@@ -626,6 +623,8 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			VersionID:    objInfo.VersionID,
 			UserDefined:  objInfo.UserDefined,
 			NoAuditLog:   true,
+			SrcPoolIdx:   idx,
+			DataMovement: true,
 		})
 		if err != nil {
 			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
@@ -660,6 +659,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
+			SrcPoolIdx:   idx,
 			DataMovement: true,
 			MTime:        objInfo.ModTime,
 			NoAuditLog:   true,
@@ -681,6 +681,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		NewPutObjReader(hr),
 		ObjectOptions{
 			DataMovement: true,
+			SrcPoolIdx:   idx,
 			VersionID:    objInfo.VersionID,
 			MTime:        objInfo.ModTime,
 			UserDefined:  objInfo.UserDefined,
@@ -855,6 +856,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					versionID = nullVersionID
 				}
 
+				var failure, ignore bool
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
 						bi.Name,
@@ -867,14 +869,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID:          versionID,
 							MTime:              version.ModTime,
 							DeleteReplication:  version.ReplicationState,
+							SrcPoolIdx:         idx,
+							DataMovement:       true,
 							DeleteMarker:       true, // make sure we create a delete marker
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 							NoAuditLog:         true,
 						})
-					var failure bool
 					if err != nil {
-						if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+						// This can happen when rebalance stop races with ongoing rebalance workers.
-							err = nil
+						// These rebalance failures can be ignored.
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							continue
 						}
 					}
 					stopFn(version.Size, err)
@@ -893,7 +900,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					continue
 				}
 
-				var failure, ignore bool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
 					if version.IsRemote() {
@@ -901,14 +907,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID:    versionID,
 							MTime:        version.ModTime,
 							UserDefined:  version.Metadata,
+							SrcPoolIdx:   idx,
+							DataMovement: true,
 						}); err != nil {
-							stopFn(version.Size, err)
+							if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
-							failure = true
+								ignore = true
-							decomLogIf(ctx, err)
+								stopFn(0, nil)
-							continue
+							}
+						}
+						if !ignore {
+							stopFn(version.Size, err)
+							failure = err != nil
+							decomLogIf(ctx, err)
 						}
-						stopFn(version.Size, nil)
-						failure = false
 						break
 					}
 					gr, err := set.GetObjectNInfo(ctx,
@@ -925,7 +936,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
-						stopFn(version.Size, nil)
+						stopFn(0, nil)
 						break
 					}
 					if err != nil && !ignore {
@@ -943,7 +954,12 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						stopFn(version.Size, err)
 						continue
 					}
-					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
+					if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil {
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							break
+						}
 						stopFn(version.Size, err)
 						failure = true
 						decomLogIf(ctx, err)

cmd/erasure-server-pool-rebalance.go

@@ -110,19 +110,14 @@ var errRebalanceNotStarted = errors.New("rebalance not started")
 
 func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error {
 	r := &rebalanceMeta{}
-	err := r.load(ctx, z.serverPools[0])
+	if err := r.load(ctx, z.serverPools[0]); err == nil {
-	if err != nil {
-		if errors.Is(err, errConfigNotFound) {
-			return nil
-		}
-		return err
-	}
-
 		z.rebalMu.Lock()
 		z.rebalMeta = r
 		z.updateRebalanceStats(ctx)
 		z.rebalMu.Unlock()
-
+	} else if !errors.Is(err, errConfigNotFound) {
+		rebalanceLogIf(ctx, fmt.Errorf("failed to load rebalance metadata, continue to restart rebalance as needed: %w", err))
+	}
 	return nil
 }
 
@@ -626,14 +621,18 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 
 			var rebalanced, expired int
 			for _, version := range fivs.Versions {
+				stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
+
 				// Skip transitioned objects for now. TBD
 				if version.IsRemote() {
+					stopFn(version.Size, errors.New("ILM Tiered version will be skipped for now"))
 					continue
 				}
 
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bucket, version.Name, version) {
 					expired++
+					stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
 					continue
 				}
 
@@ -643,6 +642,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				remainingVersions := len(fivs.Versions) - expired
 				if version.Deleted && remainingVersions == 1 {
 					rebalanced++
+					stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
 					continue
 				}
 
@@ -651,6 +651,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					versionID = nullVersionID
 				}
 
+				var failure, ignore bool
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
 						bucket,
@@ -660,16 +661,26 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 							VersionID:         versionID,
 							MTime:             version.ModTime,
 							DeleteReplication: version.ReplicationState,
+							SrcPoolIdx:        poolIdx,
+							DataMovement:      true,
 							DeleteMarker:      true, // make sure we create a delete marker
 							SkipRebalancing:   true, // make sure we skip the decommissioned pool
 							NoAuditLog:        true,
 						})
-					var failure bool
+					// This can happen when rebalance stop races with ongoing rebalance workers.
-					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+					// These rebalance failures can be ignored.
-						rebalanceLogIf(ctx, err)
+					if err != nil {
-						failure = true
+						// This can happen when rebalance stop races with ongoing rebalance workers.
+						// These rebalance failures can be ignored.
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							continue
 						}
-
+					}
+					stopFn(version.Size, err)
+					rebalanceLogIf(ctx, err)
+					failure = err != nil
 					if !failure {
 						z.updatePoolStats(poolIdx, bucket, version)
 						rebalanced++
@@ -678,10 +689,8 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					continue
 				}
 
-				var failure, ignore bool
 				for try := 0; try < 3; try++ {
 					// GetObjectReader.Close is called by rebalanceObject
-					stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
 					gr, err := set.GetObjectNInfo(ctx,
 						bucket,
 						encodeDirObject(version.Name),
@@ -706,7 +715,14 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						continue
 					}
 
-					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
+					if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil {
+						// This can happen when rebalance stop races with ongoing rebalance workers.
+						// These rebalance failures can be ignored.
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							break
+						}
 						failure = true
 						rebalanceLogIf(ctx, err)
 						stopFn(version.Size, err)
@@ -822,7 +838,7 @@ func auditLogRebalance(ctx context.Context, apiName, bucket, object, versionID s
 	})
 }
 
-func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, bucket string, gr *GetObjectReader) (err error) {
 	oi := gr.ObjInfo
 
 	defer func() {
@@ -840,6 +856,8 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 			VersionID:    oi.VersionID,
 			UserDefined:  oi.UserDefined,
 			NoAuditLog:   true,
+			DataMovement: true,
+			SrcPoolIdx:   poolIdx,
 		})
 		if err != nil {
 			return fmt.Errorf("rebalanceObject: NewMultipartUpload() %w", err)
@@ -891,6 +909,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		oi.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
+			SrcPoolIdx:   poolIdx,
 			DataMovement: true,
 			VersionID:    oi.VersionID,
 			MTime:        oi.ModTime,
@@ -981,6 +1000,8 @@ const (
 	rebalanceMetricSaveMetadata
 )
 
+var errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same")
+
 func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {

cmd/erasure-server-pool.go

@@ -45,6 +45,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/sync/errgroup"
 	"github.com/minio/pkg/v3/wildcard"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 type erasureServerPools struct {
@@ -63,6 +64,8 @@ type erasureServerPools struct {
 	decommissionCancelers []context.CancelFunc
 
 	s3Peer *S3PeerSys
+
+	mpCache *xsync.MapOf[string, MultipartInfo]
 }
 
 func (z *erasureServerPools) SinglePool() bool {
@@ -187,19 +190,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		globalLeaderLock = newSharedLock(GlobalContext, z, "leader.lock")
 	})
 
-	// Enable background operations on
+	// Start self healing after the object initialization
-	//
+	// so various tasks will be useful
-	// - Disk auto healing
-	// - MRF (most recently failed) healing
-	// - Background expiration routine for lifecycle policies
 	bootstrapTrace("initAutoHeal", func() {
 		initAutoHeal(GlobalContext, z)
 	})
 
-	bootstrapTrace("initHealMRF", func() {
-		go globalMRFState.healRoutine(z)
-	})
-
 	// initialize the object layer.
 	defer setObjectLayer(z)
 
@@ -223,15 +219,39 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		break
 	}
 
+	// initialize the incomplete uploads cache
+	z.mpCache = xsync.NewMapOf[string, MultipartInfo]()
+
+	go z.cleanupStaleMPCache(ctx)
+
 	return z, nil
 }
 
-func (z *erasureServerPools) NewNSLock(bucket string, objects ...string) RWLocker {
+func (z *erasureServerPools) cleanupStaleMPCache(ctx context.Context) {
-	poolID := hashKey(z.distributionAlgo, "", len(z.serverPools), z.deploymentID)
+	timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
-	if len(objects) >= 1 {
+	defer timer.Stop()
-		poolID = hashKey(z.distributionAlgo, objects[0], len(z.serverPools), z.deploymentID)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			z.mpCache.Range(func(id string, info MultipartInfo) bool {
+				if time.Since(info.Initiated) >= globalAPIConfig.getStaleUploadsExpiry() {
+					z.mpCache.Delete(id)
+					// No need to notify to peers, each node will delete its own cache.
 				}
-	return z.serverPools[poolID].NewNSLock(bucket, objects...)
+				return true
+			})
+
+			// Reset for the next interval
+			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
+		}
+	}
+}
+
+func (z *erasureServerPools) NewNSLock(bucket string, objects ...string) RWLocker {
+	return z.serverPools[0].NewNSLock(bucket, objects...)
 }
 
 // GetDisksID will return disks by their ID.
@@ -536,7 +556,7 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 			return pinfo, z.poolsWithObject(poolObjInfos, opts), nil
 		}
 		defPool = pinfo
-		if !isErrObjectNotFound(pinfo.Err) {
+		if !isErrObjectNotFound(pinfo.Err) && !isErrVersionNotFound(pinfo.Err) {
 			return pinfo, noReadQuorumPools, pinfo.Err
 		}
 
@@ -1071,23 +1091,20 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
 
-	if !opts.NoLock {
+	idx, err := z.getPoolIdx(ctx, bucket, object, data.Size())
-		ns := z.NewNSLock(bucket, object)
-		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
-		if err != nil {
-			return ObjectInfo{}, err
-		}
-		ctx = lkctx.Context()
-		defer ns.Unlock(lkctx)
-		opts.NoLock = true
-	}
-
-	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, data.Size())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 
-	// Overwrite the object at the right pool
+	if opts.DataMovement && idx == opts.SrcPoolIdx {
+		return ObjectInfo{}, DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
 	return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
 }
 
@@ -1139,14 +1156,45 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return pinfo.ObjInfo, nil
 	}
 
+	// Datamovement must never be allowed on the same pool.
+	if opts.DataMovement && opts.SrcPoolIdx == pinfo.Index {
+		return pinfo.ObjInfo, DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
+	if opts.DataMovement {
+		objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
+		objInfo.Name = decodeDirObject(object)
+		return objInfo, err
+	}
+
 	// Delete concurrently in all server pools with read quorum error for unversioned objects.
 	if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended {
 		return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools)
 	}
-	objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
+
+	for _, pool := range z.serverPools {
+		objInfo, err := pool.DeleteObject(ctx, bucket, object, opts)
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 			objInfo.Name = decodeDirObject(object)
 			return objInfo, err
 		}
+		if err == nil {
+			objInfo.Name = decodeDirObject(object)
+			return objInfo, nil
+		}
+	}
+
+	objInfo.Name = decodeDirObject(object)
+	if opts.VersionID != "" {
+		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+	}
+	return objInfo, ObjectNotFound{Bucket: bucket, Object: object}
+}
 
 func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) {
 	derrs := make([]error, len(poolIndices))
@@ -1264,7 +1312,9 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 			go func(idx int, pool *erasureSets) {
 				defer wg.Done()
 				objs := poolObjIdxMap[idx]
-				if len(objs) > 0 {
+				if len(objs) == 0 {
+					return
+				}
 				orgIndexes := origIndexMap[idx]
 				deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
 				mu.Lock()
@@ -1276,7 +1326,6 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 					dobjects[orgIndexes[i]] = deletedObjects[i]
 				}
 				mu.Unlock()
-				}
 			}(idx, pool)
 		}
 		wg.Wait()
@@ -1695,15 +1744,32 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 		return ListMultipartsInfo{}, err
 	}
 
-	if z.SinglePool() {
-		return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
-	}
-
 	poolResult := ListMultipartsInfo{}
 	poolResult.MaxUploads = maxUploads
 	poolResult.KeyMarker = keyMarker
 	poolResult.Prefix = prefix
 	poolResult.Delimiter = delimiter
+
+	// if no prefix provided, return the list from cache
+	if prefix == "" {
+		if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+			return ListMultipartsInfo{}, toObjectErr(err, bucket)
+		}
+
+		z.mpCache.Range(func(_ string, mp MultipartInfo) bool {
+			poolResult.Uploads = append(poolResult.Uploads, mp)
+			return true
+		})
+		sort.Slice(poolResult.Uploads, func(i int, j int) bool {
+			return poolResult.Uploads[i].Initiated.Before(poolResult.Uploads[j].Initiated)
+		})
+		return poolResult, nil
+	}
+
+	if z.SinglePool() {
+		return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	}
+
 	for idx, pool := range z.serverPools {
 		if z.IsSuspended(idx) {
 			continue
@@ -1715,15 +1781,27 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 		}
 		poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
 	}
+
 	return poolResult, nil
 }
 
 // Initiate a new multipart upload on a hashedSet based on object name.
-func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
+func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (mp *NewMultipartUploadResult, err error) {
 	if err := checkNewMultipartArgs(ctx, bucket, object); err != nil {
 		return nil, err
 	}
 
+	defer func() {
+		if err == nil && mp != nil {
+			z.mpCache.Store(mp.UploadID, MultipartInfo{
+				Bucket:    bucket,
+				Object:    object,
+				UploadID:  mp.UploadID,
+				Initiated: time.Now(),
+			})
+		}
+	}()
+
 	if z.SinglePool() {
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
@@ -1752,6 +1830,15 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 		return nil, err
 	}
 
+	if opts.DataMovement && idx == opts.SrcPoolIdx {
+		return nil, DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
 	return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
 }
 
@@ -1771,6 +1858,16 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		return PartInfo{}, err
 	}
 
+	// Read lock for upload id.
+	// Only held while reading the upload metadata.
+	uploadIDRLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return PartInfo{}, err
+	}
+	ctx = rlkctx.Context()
+	defer uploadIDRLock.RUnlock(rlkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
@@ -1803,9 +1900,18 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		return MultipartInfo{}, err
 	}
 
+	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return MultipartInfo{}, err
+	}
+	ctx = lkctx.Context()
+	defer uploadIDLock.RUnlock(lkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
+
 	for idx, pool := range z.serverPools {
 		if z.IsSuspended(idx) {
 			continue
@@ -1821,6 +1927,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		// any other unhandled error return right here.
 		return MultipartInfo{}, err
 	}
+
 	return MultipartInfo{}, InvalidUploadID{
 		Bucket:   bucket,
 		Object:   object,
@@ -1834,9 +1941,18 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		return ListPartsInfo{}, err
 	}
 
+	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return ListPartsInfo{}, err
+	}
+	ctx = lkctx.Context()
+	defer uploadIDLock.RUnlock(lkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
+
 	for idx, pool := range z.serverPools {
 		if z.IsSuspended(idx) {
 			continue
@@ -1850,6 +1966,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		}
 		return ListPartsInfo{}, err
 	}
+
 	return ListPartsInfo{}, InvalidUploadID{
 		Bucket:   bucket,
 		Object:   object,
@@ -1858,11 +1975,26 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
-func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
 	if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID); err != nil {
 		return err
 	}
 
+	defer func() {
+		if err == nil {
+			z.mpCache.Delete(uploadID)
+			globalNotificationSys.DeleteUploadID(ctx, uploadID)
+		}
+	}()
+
+	lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
@@ -1894,6 +2026,23 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		return objInfo, err
 	}
 
+	defer func() {
+		if err == nil {
+			z.mpCache.Delete(uploadID)
+			globalNotificationSys.DeleteUploadID(ctx, uploadID)
+		}
+	}()
+
+	// Hold write locks to verify uploaded parts, also disallows any
+	// parallel PutObjectPart() requests.
+	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return objInfo, err
+	}
+	ctx = wlkctx.Context()
+	defer uploadIDLock.Unlock(wlkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
@@ -1936,6 +2085,12 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, o
 	return bucketInfo, nil
 }
 
+// ClearUploadID deletes given uploadID from cache
+func (z *erasureServerPools) ClearUploadID(uploadID string) error {
+	z.mpCache.Delete(uploadID)
+	return nil
+}
+
 // DeleteBucket - deletes a bucket on all serverPools simultaneously,
 // even if one of the serverPools fail to delete buckets, we proceed to
 // undo a successful operation.
@@ -1960,6 +2115,34 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 		defer lk.Unlock(lkctx)
 	}
 
+	if !opts.Force {
+		results := make(chan itemOrErr[ObjectInfo])
+
+		ctx, cancel := context.WithTimeout(ctx, time.Minute)
+		defer cancel()
+		err := z.Walk(ctx, bucket, "", results, WalkOptions{Limit: 1})
+		if err != nil {
+			s3LogIf(ctx, fmt.Errorf("unable to verify if the bucket %s is empty: %w", bucket, err))
+			return toObjectErr(err, bucket)
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case r, found := <-results:
+			if found {
+				if r.Err != nil {
+					s3LogIf(ctx, fmt.Errorf("unable to verify if the bucket %s is empty: %w", bucket, r.Err))
+					return toObjectErr(r.Err, bucket)
+				}
+				return toObjectErr(errVolumeNotEmpty, bucket)
+			}
+		}
+
+		// Always pass force to the lower level
+		opts.Force = true
+	}
+
 	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
 	if err == nil || isErrBucketNotFound(err) {
 		// If site replication is configured, hold on to deleted bucket state until sites sync
@@ -2102,7 +2285,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 			disks, infos, _ := set.getOnlineDisksWithHealingAndInfo(true)
 			if len(disks) == 0 {
 				xioutil.SafeClose(results)
-				err := fmt.Errorf("Walk: no online disks found in pool %d, set %d", setIdx, poolIdx)
+				err := fmt.Errorf("Walk: no online disks found in (set:%d pool:%d) %w", setIdx, poolIdx, errErasureReadQuorum)
 				cancelCause(err)
 				return err
 			}
@@ -2178,6 +2361,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 					filterPrefix:   filterPrefix,
 					recursive:      true,
 					forwardTo:      opts.Marker,
+					perDiskLimit:   opts.Limit,
 					minDisks:       listingQuorum,
 					reportNotFound: false,
 					agreed:         send,
@@ -2638,7 +2822,19 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
 		return z.serverPools[0].PutObjectMetadata(ctx, bucket, object, opts)
 	}
 
+	if !opts.NoLock {
+		// Lock the object before updating metadata.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	opts.MetadataChg = true
+	opts.NoLock = true
 	// We don't know the size here set 1GiB at least.
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
@@ -2655,7 +2851,19 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
 
+	if !opts.NoLock {
+		// Lock the object before updating tags.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	opts.MetadataChg = true
+	opts.NoLock = true
 
 	// We don't know the size here set 1GiB at least.
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
@@ -2673,8 +2881,19 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 
-	opts.MetadataChg = true
+	if !opts.NoLock {
+		// Lock the object before deleting tags.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
 
+	opts.MetadataChg = true
+	opts.NoLock = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2705,8 +2924,20 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
+	if !opts.NoLock {
+		// Acquire write lock before starting to transition the object.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	// Avoid transitioning an object from a pool being decommissioned.
 	opts.SkipDecommissioned = true
+	opts.NoLock = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
@@ -2722,8 +2953,20 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
+	if !opts.NoLock {
+		// Acquire write lock before restoring transitioned object
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	// Avoid restoring object from a pool being decommissioned.
 	opts.SkipDecommissioned = true
+	opts.NoLock = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
@@ -2780,5 +3023,14 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
 		return err
 	}
 
+	if opts.DataMovement && idx == opts.SrcPoolIdx {
+		return DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
 	return z.serverPools[idx].DecomTieredObject(ctx, bucket, object, fi, opts)
 }

cmd/erasure-sets.go

@@ -86,6 +86,8 @@ type erasureSets struct {
 	lastConnectDisksOpTime time.Time
 }
 
+var staleUploadsCleanupIntervalChangedCh = make(chan struct{})
+
 func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 	diskMap := make(map[Endpoint]StorageAPI)
 
@@ -230,9 +232,12 @@ func (s *erasureSets) connectDisks(log bool) {
 				}
 				return
 			}
-			if disk.IsLocal() && disk.Healing() != nil {
+			if disk.IsLocal() {
+				h := disk.Healing()
+				if h != nil && !h.Finished {
 					globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
 				}
+			}
 			s.erasureDisksMu.Lock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
 			if err != nil {
@@ -529,22 +534,27 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 				}(set)
 			}
 			wg.Wait()
+		case <-staleUploadsCleanupIntervalChangedCh:
+		}
 
 		// Reset for the next interval
 		timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
 	}
 }
-}
 
 type auditObjectOp struct {
 	Name string `json:"name"`
 	Pool int    `json:"poolId"`
 	Set  int    `json:"setId"`
-	Drives []string `json:"drives"`
+}
+
+func (op auditObjectOp) String() string {
+	// Flatten the auditObjectOp
+	return fmt.Sprintf("name=%s,pool=%d,set=%d", op.Name, op.Pool, op.Set)
 }
 
 // Add erasure set information to the current context
-func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjects) {
+func auditObjectErasureSet(ctx context.Context, api, object string, set *erasureObjects) {
 	if len(logger.AuditTargets()) == 0 {
 		return
 	}
@@ -553,18 +563,14 @@ func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjec
 		Name: decodeDirObject(object),
 		Pool: set.poolIndex + 1,
 		Set:  set.setIndex + 1,
-		Drives: set.getEndpointStrings(),
 	}
 
-	logger.GetReqInfo(ctx).AppendTags("objectLocation", op)
+	logger.GetReqInfo(ctx).AppendTags(api, op.String())
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
 func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
-	if len(objects) == 1 {
+	return s.sets[0].NewNSLock(bucket, objects...)
-		return s.getHashedSet(objects[0]).NewNSLock(bucket, objects...)
-	}
-	return s.getHashedSet("").NewNSLock(bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.

cmd/erasure.go

@@ -90,6 +90,11 @@ func (er erasureObjects) defaultWQuorum() int {
 	return dataCount
 }
 
+// defaultRQuorum read quorum based on setDriveCount and defaultParityCount
+func (er erasureObjects) defaultRQuorum() int {
+	return er.setDriveCount - er.defaultParityCount
+}
+
 func diskErrToDriveState(err error) (state string) {
 	switch {
 	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
@@ -198,12 +203,10 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disks
 			di.State = diskErrToDriveState(err)
 			di.FreeInodes = info.FreeInodes
 			di.UsedInodes = info.UsedInodes
-			if info.Healing {
 			if hi := disks[index].Healing(); hi != nil {
 				hd := hi.toHealingDisk()
 				di.HealInfo = &hd
 			}
-			}
 			di.Metrics = &madmin.DiskMetrics{
 				LastMinute:              make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
 				APICalls:                make(map[string]uint64, len(info.Metrics.APICalls)),
@@ -548,9 +551,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				var root dataUsageEntry
 				if r := cache.root(); r != nil {
 					root = cache.flatten(*r)
-					if root.ReplicationStats.empty() {
-						root.ReplicationStats = nil
-					}
 				}
 				select {
 				case <-ctx.Done():

cmd/fmt-gen.go

@@ -15,8 +15,6 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build fmtgen
-
 package cmd
 
 import (
@@ -51,6 +49,7 @@ var fmtGenCmd = cli.Command{
 	Usage:  "Generate format.json files for an erasure server pool",
 	Flags:  append(fmtGenFlags, GlobalFlags...),
 	Action: fmtGenMain,
+	Hidden: true,
 	CustomHelpTemplate: `NAME:
   {{.HelpName}} - {{.Usage}}
 

cmd/fmt-gen_notsupported.go

@@ -1,24 +0,0 @@
-// Copyright (c) 2015-2024 MinIO, Inc.
-//
-// # This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-//go:build !fmtgen
-
-package cmd
-
-import "github.com/minio/cli"
-
-var fmtGenCmd cli.Command

cmd/ftp-server-driver.go

@@ -114,7 +114,7 @@ func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err e
 		TraceType: madmin.TraceFTP,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
-		FuncName:  fmt.Sprintf(s.Cmd),
+		FuncName:  s.Cmd,
 		Duration:  time.Since(startTime),
 		Path:      objPath,
 		Error:     errStr,

cmd/global-heal.go

@@ -167,6 +167,19 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		return errServerNotInitialized
 	}
 
+	started := tracker.Started
+	if started.IsZero() || started.Equal(timeSentinel) {
+		healingLogIf(ctx, fmt.Errorf("unexpected tracker healing start time found: %v", started))
+		started = time.Time{}
+	}
+
+	// Final tracer update before quitting
+	defer func() {
+		tracker.setObject("")
+		tracker.setBucket("")
+		healingLogIf(ctx, tracker.update(ctx))
+	}()
+
 	for _, bucket := range healBuckets {
 		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Log bucket healing error if any, we shall retry again.
@@ -194,7 +207,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		numHealers = uint64(v)
 	}
 
-	healingLogEvent(ctx, fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
+	healingLogEvent(ctx, "Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers)
 
 	jt, _ := workers.New(int(numHealers))
 
@@ -227,7 +240,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 	// Collect updates to tracker from concurrent healEntry calls
 	results := make(chan healEntryResult, 1000)
-	defer close(results)
+	quitting := make(chan struct{})
+	defer func() {
+		close(results)
+		<-quitting
+	}()
 
 	go func() {
 		for res := range results {
@@ -241,6 +258,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 			tracker.updateProgress(res.success, res.skipped, res.bytes)
 		}
+
+		healingLogIf(ctx, tracker.update(ctx))
+		close(quitting)
 	}()
 
 	var retErr error
@@ -428,13 +448,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 			var versionNotFound int
 			for _, version := range fivs.Versions {
-				// Ignore a version with a modtime newer than healing start time.
+				// Ignore healing a version if:
-				if version.ModTime.After(tracker.Started) {
+				// - It is uploaded after the drive healing is started
-					continue
+				// - An object that is already expired by ILM rule.
-				}
+				if !started.IsZero() && version.ModTime.After(started) || filterLifecycle(bucket, version.Name, version) {
-
-				// Apply lifecycle rules on the objects that are expired.
-				if filterLifecycle(bucket, version.Name, version) {
 					versionNotFound++
 					if !send(healEntrySkipped(uint64(version.Size))) {
 						return
@@ -442,8 +459,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					continue
 				}
 
-				var versionHealed bool
-
 				res, err := er.HealObject(ctx, bucket, encodedEntryName,
 					version.VersionID, madmin.HealOpts{
 						ScanMode: scanMode,
@@ -458,21 +473,23 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					}
 				} else {
 					// Look for the healing results
-					if res.After.Drives[tracker.DiskIndex].State == madmin.DriveStateOk {
+					if res.After.Drives[tracker.DiskIndex].State != madmin.DriveStateOk {
-						versionHealed = true
+						err = fmt.Errorf("unexpected after heal state: %s", res.After.Drives[tracker.DiskIndex].State)
 					}
 				}
 
-				if versionHealed {
+				if err == nil {
 					bgSeq.countHealed(madmin.HealItemObject)
 					result = healEntrySuccess(uint64(version.Size))
 				} else {
 					bgSeq.countFailed(madmin.HealItemObject)
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
-						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s (version-id=%s): %w",
+							bucket, version.Name, version.VersionID, err))
 					} else {
-						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w",
+							bucket, version.Name, err))
 					}
 				}
 
@@ -549,10 +566,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			healingLogIf(ctx, tracker.update(ctx))
 		}
 	}
-
-	tracker.setObject("")
-	tracker.setBucket("")
-
 	if retErr != nil {
 		return retErr
 	}

cmd/globals.go

@@ -41,7 +41,6 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
@@ -293,9 +292,6 @@ var (
 	// The global drive config
 	globalDriveConfig drive.Config
 
-	// The global cache config
-	globalCacheConfig cache.Config
-
 	// Global server's network statistics
 	globalConnStats = newConnStats()
 
@@ -384,9 +380,7 @@ var (
 	globalBackgroundHealRoutine = newHealRoutine()
 	globalBackgroundHealState   = newHealState(GlobalContext, false)
 
-	globalMRFState = mrfState{
+	globalMRFState = newMRFState()
-		opCh: make(chan partialOperation, mrfOpsQueueSize),
-	}
 
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool

cmd/handler-api.go

@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/shirou/gopsutil/v3/mem"
 
 	"github.com/minio/minio/internal/config/api"
@@ -39,7 +40,6 @@ import (
 type apiConfig struct {
 	mu sync.RWMutex
 
-	requestsDeadline       time.Duration
 	requestsPool           chan struct{}
 	clusterDeadline        time.Duration
 	listQuorum             string
@@ -62,7 +62,6 @@ type apiConfig struct {
 const (
 	cgroupV1MemLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
 	cgroupV2MemLimitFile = "/sys/fs/cgroup/memory.max"
-	cgroupMemNoLimit     = 9223372036854771712
 )
 
 func cgroupMemLimit() (limit uint64) {
@@ -79,10 +78,8 @@ func cgroupMemLimit() (limit uint64) {
 		// but still, no need to interpret more
 		return 0
 	}
-	if limit == cgroupMemNoLimit {
+	if limit >= 100*humanize.TiByte {
-		// No limit set, It's the highest positive signed 64-bit
+		// No limit set, or unreasonably high. Ignore
-		// integer (2^63-1), rounded down to multiples of 4096 (2^12),
-		// the most common page size on x86 systems - for cgroup_limits.
 		return 0
 	}
 	return limit
@@ -164,15 +161,14 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 		// but this shouldn't last long.
 		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
 	}
-	t.requestsDeadline = cfg.RequestsDeadline
 	listQuorum := cfg.ListQuorum
 	if listQuorum == "" {
 		listQuorum = "strict"
 	}
 	t.listQuorum = listQuorum
-	if globalReplicationPool != nil &&
+	if r := globalReplicationPool.GetNonBlocking(); r != nil &&
 		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers || cfg.ReplicationMaxLWorkers != t.replicationMaxLWorkers) {
-		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers)
+		r.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers)
 	}
 	t.replicationPriority = cfg.ReplicationPriority
 	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
@@ -185,13 +181,22 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 	t.transitionWorkers = cfg.TransitionWorkers
 
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
-	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
 	t.enableODirect = cfg.EnableODirect
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
 	t.syncEvents = cfg.SyncEvents
 	t.objectMaxVersions = cfg.ObjectMaxVersions
+
+	if t.staleUploadsCleanupInterval != cfg.StaleUploadsCleanupInterval {
+		t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
+
+		// signal that cleanup interval has changed
+		select {
+		case staleUploadsCleanupIntervalChangedCh <- struct{}{}:
+		default: // in case the channel is blocked...
+		}
+	}
 }
 
 func (t *apiConfig) odirectEnabled() bool {
@@ -290,19 +295,15 @@ func (t *apiConfig) getRequestsPoolCapacity() int {
 	return cap(t.requestsPool)
 }
 
-func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
+func (t *apiConfig) getRequestsPool() chan struct{} {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
 	if t.requestsPool == nil {
-		return nil, 10 * time.Second
+		return nil
 	}
 
-	if t.requestsDeadline <= 0 {
+	return t.requestsPool
-		return t.requestsPool, 10 * time.Second
-	}
-
-	return t.requestsPool, t.requestsDeadline
 }
 
 // maxClients throttles the S3 API calls
@@ -325,27 +326,19 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		}
 
 		globalHTTPStats.addRequestsInQueue(1)
-		pool, deadline := globalAPIConfig.getRequestsPool()
+		pool := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			globalHTTPStats.addRequestsInQueue(-1)
 			f.ServeHTTP(w, r)
 			return
 		}
 
-		// No deadline to wait, there is nothing to queue
-		// perform the API call immediately.
-		if deadline <= 0 {
-			globalHTTPStats.addRequestsInQueue(-1)
-			f.ServeHTTP(w, r)
-			return
-		}
-
 		if tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt); ok {
 			tc.FuncName = "s3.MaxClients"
 		}
 
-		deadlineTimer := time.NewTimer(deadline)
+		w.Header().Set("X-RateLimit-Limit", strconv.Itoa(cap(pool)))
-		defer deadlineTimer.Stop()
+		w.Header().Set("X-RateLimit-Remaining", strconv.Itoa(cap(pool)-len(pool)))
 
 		ctx := r.Context()
 		select {
@@ -357,7 +350,13 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 				return
 			}
 			f.ServeHTTP(w, r)
-		case <-deadlineTimer.C:
+		case <-r.Context().Done():
+			globalHTTPStats.addRequestsInQueue(-1)
+			// When the client disconnects before getting the S3 handler
+			// status code response, set the status code to 499 so this request
+			// will be properly audited and traced.
+			w.WriteHeader(499)
+		default:
 			globalHTTPStats.addRequestsInQueue(-1)
 			if contextCanceled(ctx) {
 				w.WriteHeader(499)
@@ -367,12 +366,7 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 			writeErrorResponse(ctx, w,
 				errorCodes.ToAPIErr(ErrTooManyRequests),
 				r.URL)
-		case <-r.Context().Done():
+
-			globalHTTPStats.addRequestsInQueue(-1)
-			// When the client disconnects before getting the S3 handler
-			// status code response, set the status code to 499 so this request
-			// will be properly audited and traced.
-			w.WriteHeader(499)
 		}
 	}
 }

cmd/http-stats.go

@@ -133,7 +133,7 @@ func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.Response
 		bucketHTTPRequestsDuration.With(prometheus.Labels{
 			"api":    api,
 			"bucket": bucket,
-		}).Observe(w.TimeToFirstByte.Seconds())
+		}).Observe(w.TTFB().Seconds())
 	}
 
 	bh.Lock()
@@ -433,7 +433,7 @@ func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label
-	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
+	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TTFB().Seconds())
 
 	code := w.StatusCode
 

cmd/http-tracer.go

@@ -164,7 +164,7 @@ func httpTracerMiddleware(h http.Handler) http.Handler {
 					Latency:         reqEndTime.Sub(respRecorder.StartTime),
 					InputBytes:      inputBytes,
 					OutputBytes:     respRecorder.Size(),
-					TimeToFirstByte: respRecorder.TimeToFirstByte,
+					TimeToFirstByte: respRecorder.TTFB(),
 				},
 			},
 		}

cmd/iam-etcd-store.go

@@ -249,6 +249,18 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	return nil
 }
 
+func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) {
+	var u UserIdentity
+	err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return "", errNoSuchUser
+		}
+		return "", err
+	}
+	return u.Credentials.SecretKey, nil
+}
+
 func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	var u UserIdentity
 	err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))

cmd/iam-object-store.go

@@ -34,6 +34,7 @@ import (
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/sync/errgroup"
 	"github.com/puzpuzpuz/xsync/v3"
 )
 
@@ -182,19 +183,20 @@ func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy
 	}
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (PolicyDoc, error) {
+	var p PolicyDoc
+
 	data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy))
 	if err != nil {
 		if err == errConfigNotFound {
-			return errNoSuchPolicy
+			return p, errNoSuchPolicy
 		}
-		return err
+		return p, err
 	}
 
-	var p PolicyDoc
 	err = p.parseJSON(data)
 	if err != nil {
-		return err
+		return p, err
 	}
 
 	if p.Version == 0 {
@@ -205,6 +207,14 @@ func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m
 		p.UpdateDate = objInfo.ModTime
 	}
 
+	return p, nil
+}
+
+func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+	p, err := iamOS.loadPolicy(ctx, policy)
+	if err != nil {
+		return err
+	}
 	m[policy] = p
 	return nil
 }
@@ -225,21 +235,33 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]Po
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
+func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) {
+	var u UserIdentity
+	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return "", errNoSuchUser
+		}
+		return "", err
+	}
+	return u.Credentials.SecretKey, nil
+}
+
+func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string, userType IAMUserType) (UserIdentity, error) {
 	var u UserIdentity
 	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
 	if err != nil {
 		if err == errConfigNotFound {
-			return errNoSuchUser
+			return u, errNoSuchUser
 		}
-		return err
+		return u, err
 	}
 
 	if u.Credentials.IsExpired() {
 		// Delete expired identity - ignoring errors here.
 		iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
 		iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
-		return nil
+		return u, errNoSuchUser
 	}
 
 	if u.Credentials.AccessKey == "" {
@@ -255,7 +277,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
 				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
 			}
-			return nil
+			return u, errNoSuchUser
 
 		}
 		u.Credentials.Claims = jwtClaims.Map()
@@ -265,6 +287,35 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 		u.Credentials.Description = u.Credentials.Comment
 	}
 
+	return u, nil
+}
+
+func (iamOS *IAMObjectStore) loadUserConcurrent(ctx context.Context, userType IAMUserType, users ...string) ([]UserIdentity, error) {
+	userIdentities := make([]UserIdentity, len(users))
+	g := errgroup.WithNErrs(len(users))
+
+	for index := range users {
+		index := index
+		g.Go(func() error {
+			userName := path.Dir(users[index])
+			user, err := iamOS.loadUserIdentity(ctx, userName, userType)
+			if err != nil && !errors.Is(err, errNoSuchUser) {
+				return fmt.Errorf("unable to load the user `%s`: %w", userName, err)
+			}
+			userIdentities[index] = user
+			return nil
+		}, index)
+	}
+
+	err := errors.Join(g.Wait()...)
+	return userIdentities, err
+}
+
+func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
+	u, err := iamOS.loadUserIdentity(ctx, user, userType)
+	if err != nil {
+		return err
+	}
 	m[user] = u
 	return nil
 }
@@ -346,16 +397,44 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
 	}
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name string, userType IAMUserType, isGroup bool) (MappedPolicy, error) {
 	var p MappedPolicy
 	err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
 		if err == errConfigNotFound {
-			return errNoSuchPolicy
+			return p, errNoSuchPolicy
 		}
-		return err
+		return p, err
+	}
+	return p, nil
 }
 
+func (iamOS *IAMObjectStore) loadMappedPolicyConcurrent(ctx context.Context, userType IAMUserType, isGroup bool, users ...string) ([]MappedPolicy, error) {
+	mappedPolicies := make([]MappedPolicy, len(users))
+	g := errgroup.WithNErrs(len(users))
+
+	for index := range users {
+		index := index
+		g.Go(func() error {
+			userName := strings.TrimSuffix(users[index], ".json")
+			userMP, err := iamOS.loadMappedPolicyInternal(ctx, userName, userType, isGroup)
+			if err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return fmt.Errorf("unable to load the user policy map `%s`: %w", userName, err)
+			}
+			mappedPolicies[index] = userMP
+			return nil
+		}, index)
+	}
+
+	err := errors.Join(g.Wait()...)
+	return mappedPolicies, err
+}
+
+func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+	p, err := iamOS.loadMappedPolicyInternal(ctx, name, userType, isGroup)
+	if err != nil {
+		return err
+	}
 	m.Store(name, p)
 	return nil
 }
@@ -402,12 +481,24 @@ var (
 	policyDBGroupsListKey   = "policydb/groups/"
 )
 
+func findSecondIndex(s string, substr string) int {
+	first := strings.Index(s, substr)
+	if first == -1 {
+		return -1
+	}
+	second := strings.Index(s[first+1:], substr)
+	if second == -1 {
+		return -1
+	}
+	return first + second + 1
+}
+
 // splitPath splits a path into a top-level directory and a child item. The
 // parent directory retains the trailing slash.
-func splitPath(s string, lastIndex bool) (string, string) {
+func splitPath(s string, secondIndex bool) (string, string) {
 	var i int
-	if lastIndex {
+	if secondIndex {
-		i = strings.LastIndex(s, "/")
+		i = findSecondIndex(s, "/")
 	} else {
 		i = strings.Index(s, "/")
 	}
@@ -427,8 +518,8 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (res map
 			return nil, item.Err
 		}
 
-		lastIndex := strings.HasPrefix(item.Item, policyDBPrefix)
+		secondIndex := strings.HasPrefix(item.Item, policyDBPrefix)
-		listKey, trimmedItem := splitPath(item.Item, lastIndex)
+		listKey, trimmedItem := splitPath(item.Item, secondIndex)
 		if listKey == iamFormatFile {
 			continue
 		}
@@ -443,6 +534,27 @@ const (
 	maxIAMLoadOpTime = 5 * time.Second
 )
 
+func (iamOS *IAMObjectStore) loadPolicyDocConcurrent(ctx context.Context, policies ...string) ([]PolicyDoc, error) {
+	policyDocs := make([]PolicyDoc, len(policies))
+	g := errgroup.WithNErrs(len(policies))
+
+	for index := range policies {
+		index := index
+		g.Go(func() error {
+			policyName := path.Dir(policies[index])
+			policyDoc, err := iamOS.loadPolicy(ctx, policyName)
+			if err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err)
+			}
+			policyDocs[index] = policyDoc
+			return nil
+		}, index)
+	}
+
+	err := errors.Join(g.Wait()...)
+	return policyDocs, err
+}
+
 // Assumes cache is locked by caller.
 func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache, firstTime bool) error {
 	bootstrapTraceMsgFirstTime := func(s string) {
@@ -478,12 +590,37 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 
 	policyLoadStartTime := UTCNow()
 	policiesList := listedConfigItems[policiesListKey]
-	for _, item := range policiesList {
+	count := 32 // number of parallel IAM loaders
-		policyName := path.Dir(item)
+	for {
-		if err := iamOS.loadPolicyDoc(ctx, policyName, cache.iamPolicyDocsMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if len(policiesList) < count {
-			return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err)
+			policyDocs, err := iamOS.loadPolicyDocConcurrent(ctx, policiesList...)
+			if err != nil {
+				return err
+			}
+			for index := range policiesList {
+				if policyDocs[index].Policy.Version != "" {
+					policyName := path.Dir(policiesList[index])
+					cache.iamPolicyDocsMap[policyName] = policyDocs[index]
 				}
 			}
+			break
+		}
+
+		policyDocs, err := iamOS.loadPolicyDocConcurrent(ctx, policiesList[:count]...)
+		if err != nil {
+			return err
+		}
+
+		for index := range policiesList[:count] {
+			if policyDocs[index].Policy.Version != "" {
+				policyName := path.Dir(policiesList[index])
+				cache.iamPolicyDocsMap[policyName] = policyDocs[index]
+			}
+		}
+
+		policiesList = policiesList[count:]
+	}
+
 	if took := time.Since(policyLoadStartTime); took > maxIAMLoadOpTime {
 		logger.Info("Policy docs load took %.2fs (for %d items)", took.Seconds(), len(policiesList))
 	}
@@ -492,12 +629,37 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		bootstrapTraceMsgFirstTime("loading regular IAM users")
 		regUsersLoadStartTime := UTCNow()
 		regUsersList := listedConfigItems[usersListKey]
-		for _, item := range regUsersList {
+
-			userName := path.Dir(item)
+		for {
-			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+			if len(regUsersList) < count {
-				return fmt.Errorf("unable to load the user: %w", err)
+				users, err := iamOS.loadUserConcurrent(ctx, regUser, regUsersList...)
+				if err != nil {
+					return err
+				}
+				for index := range regUsersList {
+					if users[index].Credentials.AccessKey != "" {
+						userName := path.Dir(regUsersList[index])
+						cache.iamUsersMap[userName] = users[index]
 					}
 				}
+				break
+			}
+
+			users, err := iamOS.loadUserConcurrent(ctx, regUser, regUsersList[:count]...)
+			if err != nil {
+				return err
+			}
+
+			for index := range regUsersList[:count] {
+				if users[index].Credentials.AccessKey != "" {
+					userName := path.Dir(regUsersList[index])
+					cache.iamUsersMap[userName] = users[index]
+				}
+			}
+
+			regUsersList = regUsersList[count:]
+		}
+
 		if took := time.Since(regUsersLoadStartTime); took > maxIAMLoadOpTime {
 			actualLoaded := len(cache.iamUsersMap)
 			logger.Info("Reg. users load took %.2fs (for %d items with %d expired items)", took.Seconds(),
@@ -521,12 +683,38 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	bootstrapTraceMsgFirstTime("loading user policy mapping")
 	userPolicyMappingLoadStartTime := UTCNow()
 	userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
-	for _, item := range userPolicyMappingsList {
+	for {
-		userName := strings.TrimSuffix(item, ".json")
+		if len(userPolicyMappingsList) < count {
-		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			mappedPolicies, err := iamOS.loadMappedPolicyConcurrent(ctx, regUser, false, userPolicyMappingsList...)
-			return fmt.Errorf("unable to load the policy mapping for the user: %w", err)
+			if err != nil {
+				return err
+			}
+
+			for index := range userPolicyMappingsList {
+				if mappedPolicies[index].Policies != "" {
+					userName := strings.TrimSuffix(userPolicyMappingsList[index], ".json")
+					cache.iamUserPolicyMap.Store(userName, mappedPolicies[index])
 				}
 			}
+
+			break
+		}
+
+		mappedPolicies, err := iamOS.loadMappedPolicyConcurrent(ctx, regUser, false, userPolicyMappingsList[:count]...)
+		if err != nil {
+			return err
+		}
+
+		for index := range userPolicyMappingsList[:count] {
+			if mappedPolicies[index].Policies != "" {
+				userName := strings.TrimSuffix(userPolicyMappingsList[index], ".json")
+				cache.iamUserPolicyMap.Store(userName, mappedPolicies[index])
+			}
+		}
+
+		userPolicyMappingsList = userPolicyMappingsList[count:]
+	}
+
 	if took := time.Since(userPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
 		logger.Info("User policy mappings load took %.2fs (for %d items)", took.Seconds(), len(userPolicyMappingsList))
 	}
@@ -605,21 +793,16 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range listedConfigItems[stsListKey] {
 		userName := path.Dir(item)
 		// loadUser() will delete expired user during the load.
-		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
+		iamLogIf(ctx, iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore))
-		if err != nil && !errors.Is(err, errNoSuchUser) {
+		// No need to return errors for failed expiration of STS users
-			return fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item)
 	}
 
-	}
 	// Loading the STS policy mappings from disk ensures that stale entries
 	// (removed during loadUser() in the loop above) are removed from memory.
 	for _, item := range listedConfigItems[policyDBSTSUsersListKey] {
 		stsName := strings.TrimSuffix(item, ".json")
-		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
+		iamLogIf(ctx, iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore))
-		if err != nil && !errors.Is(err, errNoSuchPolicy) {
+		// No need to return errors for failed expiration of STS users
-			return fmt.Errorf("unable to load policies during STS purge: %w (%s)", err, item)
-		}
-
 	}
 
 	took := time.Since(purgeStart).Seconds()
@@ -631,7 +814,11 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	// Store the newly populated map in the iam cache. This takes care of
 	// removing stale entries from the existing map.
 	cache.iamSTSAccountsMap = stsAccountsFromStore
-	cache.iamSTSPolicyMap = stsAccPoliciesFromStore
+
+	stsAccPoliciesFromStore.Range(func(k string, v MappedPolicy) bool {
+		cache.iamSTSPolicyMap.Store(k, v)
+		return true
+	})
 
 	return nil
 }

cmd/iam-object-store_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+)
+
+func TestSplitPath(t *testing.T) {
+	cases := []struct {
+		path                          string
+		secondIndex                   bool
+		expectedListKey, expectedItem string
+	}{
+		{"format.json", false, "format.json", ""},
+		{"users/tester.json", false, "users/", "tester.json"},
+		{"groups/test/group.json", false, "groups/", "test/group.json"},
+		{"policydb/groups/testgroup.json", true, "policydb/groups/", "testgroup.json"},
+		{
+			"policydb/sts-users/uid=slash/user,ou=people,ou=swengg,dc=min,dc=io.json", true,
+			"policydb/sts-users/", "uid=slash/user,ou=people,ou=swengg,dc=min,dc=io.json",
+		},
+		{
+			"policydb/sts-users/uid=slash/user/twice,ou=people,ou=swengg,dc=min,dc=io.json", true,
+			"policydb/sts-users/", "uid=slash/user/twice,ou=people,ou=swengg,dc=min,dc=io.json",
+		},
+		{
+			"policydb/groups/cn=project/d,ou=groups,ou=swengg,dc=min,dc=io.json", true,
+			"policydb/groups/", "cn=project/d,ou=groups,ou=swengg,dc=min,dc=io.json",
+		},
+	}
+	for i, test := range cases {
+		listKey, item := splitPath(test.path, test.secondIndex)
+		if listKey != test.expectedListKey || item != test.expectedItem {
+			t.Errorf("unexpected result on test[%v]: expected[%s, %s] but got [%s, %s]", i, test.expectedListKey, test.expectedItem, listKey, item)
+		}
+	}
+}

cmd/iam-store.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
 	"sort"
 	"strings"
 	"time"
@@ -31,9 +32,10 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
-	"github.com/minio/pkg/v3/console"
+	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/policy"
 	"github.com/puzpuzpuz/xsync/v3"
 	"golang.org/x/sync/singleflight"
@@ -366,7 +368,7 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
 // information in IAM (i.e sys.iam*Map) - this info is stored only in the STS
 // generated credentials. Thus we skip looking up group memberships, user map,
 // and group map and check the appropriate policy maps directly.
-func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([]string, time.Time, error) {
+func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool, policyPresent bool) ([]string, time.Time, error) {
 	if isGroup {
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[name]
@@ -391,13 +393,34 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		if ok {
 			return policy.toSlice(), policy.UpdatedAt, nil
 		}
-		if err := store.loadMappedPolicyWithRetry(context.TODO(), name, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if !policyPresent {
+			if err := store.loadMappedPolicy(context.TODO(), name, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 				return nil, time.Time{}, err
 			}
 			policy, _ = c.iamGroupPolicyMap.Load(name)
 			return policy.toSlice(), policy.UpdatedAt, nil
 		}
+		return nil, time.Time{}, nil
+	}
 
+	// returned policy could be empty, we use set to de-duplicate.
+	var policies set.StringSet
+	var updatedAt time.Time
+
+	if store.getUsersSysType() == LDAPUsersSysType {
+		// For LDAP policy mapping is part of STS users, we only need to lookup
+		// those mappings.
+		mp, ok := c.iamSTSPolicyMap.Load(name)
+		if !ok {
+			// Attempt to load parent user mapping for STS accounts
+			if err := store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, time.Time{}, err
+			}
+			mp, _ = c.iamSTSPolicyMap.Load(name)
+		}
+		policies = set.CreateStringSet(mp.toSlice()...)
+		updatedAt = mp.UpdatedAt
+	} else {
 		// When looking for a user's policies, we also check if the user
 		// and the groups they are member of are enabled.
 		u, ok := c.iamUsersMap[name]
@@ -412,10 +435,9 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		// passed here and we lookup the mapping in iamSTSPolicyMap.
 		mp, ok := c.iamUserPolicyMap.Load(name)
 		if !ok {
-		if err := store.loadMappedPolicyWithRetry(context.TODO(), name, regUser, false, c.iamUserPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			if err := store.loadMappedPolicy(context.TODO(), name, regUser, false, c.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 				return nil, time.Time{}, err
 			}
-
 			mp, ok = c.iamUserPolicyMap.Load(name)
 			if !ok {
 				// Since user "name" could be a parent user of an STS account, we look up
@@ -423,30 +445,18 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 				mp, ok = c.iamSTSPolicyMap.Load(name)
 				if !ok {
 					// Attempt to load parent user mapping for STS accounts
-				if err := store.loadMappedPolicyWithRetry(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+					if err := store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 						return nil, time.Time{}, err
 					}
 					mp, _ = c.iamSTSPolicyMap.Load(name)
 				}
 			}
 		}
-
+		policies = set.CreateStringSet(mp.toSlice()...)
-	// returned policy could be empty, we use set to de-duplicate.
-	policies := set.CreateStringSet(mp.toSlice()...)
 
 		for _, group := range u.Credentials.Groups {
-		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[group]
-			if !ok {
+			if ok {
-				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
-					return nil, time.Time{}, err
-				}
-				g, ok = c.iamGroupsMap[group]
-				if !ok {
-					return nil, time.Time{}, errNoSuchGroup
-				}
-			}
-
 				// Group is disabled, so we return no policy - this
 				// ensures the request is denied.
 				if g.Status == statusDisabled {
@@ -456,7 +466,7 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 
 			policy, ok := c.iamGroupPolicyMap.Load(group)
 			if !ok {
-			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				if err := store.loadMappedPolicy(context.TODO(), group, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 					return nil, time.Time{}, err
 				}
 				policy, _ = c.iamGroupPolicyMap.Load(group)
@@ -466,30 +476,24 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 				policies.Add(p)
 			}
 		}
+		updatedAt = mp.UpdatedAt
+	}
 
 	for _, group := range c.iamUserGroupMemberships[name].ToSlice() {
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[group]
-			if !ok {
+			if ok {
-				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
-					return nil, time.Time{}, err
-				}
-				g, ok = c.iamGroupsMap[group]
-				if !ok {
-					return nil, time.Time{}, errNoSuchGroup
-				}
-			}
-
 				// Group is disabled, so we return no policy - this
 				// ensures the request is denied.
 				if g.Status == statusDisabled {
 					return nil, time.Time{}, nil
 				}
 			}
+		}
 
 		policy, ok := c.iamGroupPolicyMap.Load(group)
 		if !ok {
-			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			if err := store.loadMappedPolicy(context.TODO(), group, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 				return nil, time.Time{}, err
 			}
 			policy, _ = c.iamGroupPolicyMap.Load(group)
@@ -500,7 +504,7 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		}
 	}
 
-	return policies.ToSlice(), mp.UpdatedAt, nil
+	return policies.ToSlice(), updatedAt, nil
 }
 
 func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
@@ -535,6 +539,7 @@ type IAMStorageAPI interface {
 	loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error
 	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
 	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error
+	loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error)
 	loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error
 	loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error
 	loadGroups(ctx context.Context, m map[string]GroupInfo) error
@@ -641,7 +646,7 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
 	// An in-memory cache must be replaced only if we know for sure that the
 	// values loaded from disk are not stale. They might be stale if the
 	// cached.updatedAt is more recent than the refresh cycle began.
-	if cache.updatedAt.Before(loadedAt) {
+	if cache.updatedAt.Before(loadedAt) || firstTime {
 		// No one has updated anything since the config was loaded,
 		// so we just replace whatever is on the disk into memory.
 		cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap
@@ -752,13 +757,14 @@ func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string,
 	cache := store.rlock()
 	defer store.runlock()
 
-	policies, _, err := cache.policyDBGet(store, name, false)
+	policies, _, err := cache.policyDBGet(store, name, false, false)
 	if err != nil {
 		return nil, err
 	}
 
+	userPolicyPresent := len(policies) > 0
 	for _, group := range groups {
-		ps, _, err := cache.policyDBGet(store, group, true)
+		ps, _, err := cache.policyDBGet(store, group, true, userPolicyPresent)
 		if err != nil {
 			return nil, err
 		}
@@ -943,7 +949,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	cache := store.rlock()
 	defer store.runlock()
 
-	ps, updatedAt, err := cache.policyDBGet(store, group, true)
+	ps, updatedAt, err := cache.policyDBGet(store, group, true, false)
 	if err != nil {
 		return gd, err
 	}
@@ -972,35 +978,48 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	}, nil
 }
 
+// updateGroups updates the group from the persistent store, and also related policy mapping if any.
 func (store *IAMStoreSys) updateGroups(ctx context.Context, cache *iamCache) (res []string, err error) {
-	if store.getUsersSysType() == MinIOUsersSysType {
+	groupSet := set.NewStringSet()
-		m := map[string]GroupInfo{}
+	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
-		err = store.loadGroups(ctx, m)
+		listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 		if err != nil {
-			return
+			return nil, err
 		}
-		cache.iamGroupsMap = m
+		if store.getUsersSysType() == MinIOUsersSysType {
-		cache.updatedAt = time.Now()
+			groupsList := listedConfigItems[groupsListKey]
-		for k := range cache.iamGroupsMap {
+			for _, item := range groupsList {
-			res = append(res, k)
+				group := path.Dir(item)
+				if err = iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && !errors.Is(err, errNoSuchGroup) {
+					return nil, fmt.Errorf("unable to load the group: %w", err)
+				}
+				groupSet.Add(group)
 			}
 		}
 
-	if store.getUsersSysType() == LDAPUsersSysType {
+		groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
-		m := xsync.NewMapOf[string, MappedPolicy]()
+		for _, item := range groupPolicyMappingsList {
-		err = store.loadMappedPolicies(ctx, stsUser, true, m)
+			group := strings.TrimSuffix(item, ".json")
-		if err != nil {
+			if err = iamOS.loadMappedPolicy(ctx, group, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return
+				return nil, fmt.Errorf("unable to load the policy mapping for the group: %w", err)
 			}
-		cache.iamGroupPolicyMap = m
+			groupSet.Add(group)
-		cache.updatedAt = time.Now()
+		}
+
+		return groupSet.ToSlice(), nil
+	}
+
+	// For etcd just return from cache.
+	for k := range cache.iamGroupsMap {
+		groupSet.Add(k)
+	}
+
 	cache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool {
-			res = append(res, k)
+		groupSet.Add(k)
 		return true
 	})
-	}
 
-	return res, nil
+	return groupSet.ToSlice(), nil
 }
 
 // ListGroups - lists groups. Since this is not going to be a frequent
@@ -1862,8 +1881,12 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 					delete(cache.iamUsersMap, u.AccessKey)
 				case u.IsTemp():
 					_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
+					delete(cache.iamSTSAccountsMap, u.AccessKey)
 					delete(cache.iamUsersMap, u.AccessKey)
 				}
+				if store.group != nil {
+					store.group.Forget(u.AccessKey)
+				}
 			}
 		}
 	}
@@ -1877,8 +1900,13 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 		// ignore if user is already deleted.
 		err = nil
 	}
+	if userType == stsUser {
+		delete(cache.iamSTSAccountsMap, accessKey)
+	}
 	delete(cache.iamUsersMap, accessKey)
+	if store.group != nil {
 		store.group.Forget(accessKey)
+	}
 
 	cache.updatedAt = time.Now()
 
@@ -1953,8 +1981,13 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 			// we are only logging errors, not handling them.
 			err := store.deleteUserIdentity(ctx, user, userType)
 			iamLogIf(GlobalContext, err)
+			if userType == stsUser {
+				delete(cache.iamSTSAccountsMap, user)
+			}
 			delete(cache.iamUsersMap, user)
+			if store.group != nil {
 				store.group.Forget(user)
+			}
 
 			deleted = true
 		}
@@ -1976,7 +2009,8 @@ type ParentUserInfo struct {
 // GetAllParentUsers - returns all distinct "parent-users" associated with STS
 // or service credentials, mapped to all distinct roleARNs associated with the
 // parent user. The dummy role ARN is associated with parent users from
-// policy-claim based OpenID providers.
+// policy-claim based OpenID providers. The root credential as a parent
+// user is not included in the result.
 func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	cache := store.rlock()
 	defer store.runlock()
@@ -1998,7 +2032,7 @@ func (store *IAMStoreSys) getParentUsers(cache *iamCache) map[string]ParentUserI
 
 		var (
 			err    error
-			claims map[string]interface{} = cred.Claims
+			claims *jwt.MapClaims
 		)
 
 		if cred.IsServiceAccount() {
@@ -2015,29 +2049,22 @@ func (store *IAMStoreSys) getParentUsers(cache *iamCache) map[string]ParentUserI
 		if err != nil {
 			continue
 		}
-		if cred.ParentUser == "" {
+		if cred.ParentUser == "" || cred.ParentUser == globalActiveCred.AccessKey {
 			continue
 		}
 
 		subClaimValue := cred.ParentUser
-		if v, ok := claims[subClaim]; ok {
+		if v, ok := claims.Lookup(subClaim); ok {
-			subFromToken, ok := v.(string)
+			subClaimValue = v
-			if ok {
-				subClaimValue = subFromToken
-			}
-		}
-		if v, ok := claims[ldapActualUser]; ok {
-			subFromToken, ok := v.(string)
-			if ok {
-				subClaimValue = subFromToken
 		}
+		if v, ok := claims.Lookup(ldapActualUser); ok {
+			subClaimValue = v
 		}
 
 		roleArn := openid.DummyRoleARN.String()
-		s, ok := claims[roleArnClaim]
+		s, ok := claims.Lookup(roleArnClaim)
-		val, ok2 := s.(string)
+		if ok {
-		if ok && ok2 {
+			roleArn = s
-			roleArn = val
 		}
 		v, ok := res[cred.ParentUser]
 		if ok {
@@ -2504,13 +2531,15 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	// Extracted session policy name string can be removed as its not useful
 	// at this point.
-	delete(m, sessionPolicyNameExtracted)
+	m.Delete(sessionPolicyNameExtracted)
+
+	nosp := opts.sessionPolicy == nil || opts.sessionPolicy.Version == "" && len(opts.sessionPolicy.Statements) == 0
 
 	// sessionPolicy is nil and there is embedded policy attached we remove
 	// embedded policy at that point.
-	if _, ok := m[policy.SessionPolicyName]; ok && opts.sessionPolicy == nil {
+	if _, ok := m.Lookup(policy.SessionPolicyName); ok && nosp {
-		delete(m, policy.SessionPolicyName)
+		m.Delete(policy.SessionPolicyName)
-		m[iamPolicyClaimNameSA()] = inheritedPolicyType
+		m.Set(iamPolicyClaimNameSA(), inheritedPolicyType)
 	}
 
 	if opts.sessionPolicy != nil { // session policies is being updated
@@ -2518,6 +2547,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 			return updatedAt, err
 		}
 
+		if opts.sessionPolicy.Version != "" && len(opts.sessionPolicy.Statements) > 0 {
 			policyBuf, err := json.Marshal(opts.sessionPolicy)
 			if err != nil {
 				return updatedAt, err
@@ -2528,11 +2558,12 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 			}
 
 			// Overwrite session policy claims.
-		m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
+			m.Set(policy.SessionPolicyName, base64.StdEncoding.EncodeToString(policyBuf))
-		m[iamPolicyClaimNameSA()] = embeddedPolicyType
+			m.Set(iamPolicyClaimNameSA(), embeddedPolicyType)
+		}
 	}
 
-	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
+	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m.Map(), cr.SecretKey)
 	if err != nil {
 		return updatedAt, err
 	}
@@ -2725,115 +2756,161 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 
 // LoadUser - attempts to load user info from storage and updates cache.
 func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error {
-	// We use singleflight to de-duplicate requests when server
+	groupLoad := env.Get("_MINIO_IAM_GROUP_REFRESH", config.EnableOff) == config.EnableOn
-	// is coming up and loading accessKey and its associated assets
-	val, err, shared := store.group.Do(accessKey, func() (val interface{}, err error) {
-		cache := store.lock()
-		defer func() {
-			cache.updatedAt = time.Now()
-			store.unlock()
-		}()
 
-		_, found := cache.iamUsersMap[accessKey]
+	newCachePopulate := func() (val interface{}, err error) {
+		newCache := newIamCache()
 
-		// Check for regular user access key
+		// Check for service account first
-		if !found {
+		store.loadUser(ctx, accessKey, svcUser, newCache.iamUsersMap)
-			err = store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)
-			if _, found = cache.iamUsersMap[accessKey]; found {
-				// load mapped policies
-				err = store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, cache.iamUserPolicyMap, 3)
-			}
-		}
 
-		// Check for service account
+		svc, found := newCache.iamUsersMap[accessKey]
-		if !found {
-			err = store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
-			var svc UserIdentity
-			svc, found = cache.iamUsersMap[accessKey]
 		if found {
 			// Load parent user and mapped policies.
 			if store.getUsersSysType() == MinIOUsersSysType {
-					err = store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
+				err = store.loadUser(ctx, svc.Credentials.ParentUser, regUser, newCache.iamUsersMap)
-					if err == nil {
+				// NOTE: we are not worried about loading errors from policies.
-						err = store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap, 3)
+				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, newCache.iamUserPolicyMap, 3)
-					}
 			} else {
-					// In case of LDAP the parent user's policy mapping needs to be
+				// In case of LDAP the parent user's policy mapping needs to be loaded into sts map
-					// loaded into sts map
+				// NOTE: we are not worried about loading errors from policies.
-					err = store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
+				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, newCache.iamSTSPolicyMap, 3)
 			}
 		}
+
+		if !found {
+			err = store.loadUser(ctx, accessKey, regUser, newCache.iamUsersMap)
+			if _, found = newCache.iamUsersMap[accessKey]; found {
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, newCache.iamUserPolicyMap, 3)
+			}
 		}
 
 		// Check for STS account
-		stsAccountFound := false
 		var stsUserCred UserIdentity
 		if !found {
-			err = store.loadUser(ctx, accessKey, stsUser, cache.iamSTSAccountsMap)
+			err = store.loadUser(ctx, accessKey, stsUser, newCache.iamSTSAccountsMap)
-			if stsUserCred, found = cache.iamSTSAccountsMap[accessKey]; found {
+			if stsUserCred, found = newCache.iamSTSAccountsMap[accessKey]; found {
 				// Load mapped policy
-				err = store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
+				// NOTE: we are not worried about loading errors from policies.
-				stsAccountFound = true
+				store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, newCache.iamSTSPolicyMap, 3)
 			}
 		}
 
 		// Load any associated policy definitions
-		if !stsAccountFound {
+		pols, _ := newCache.iamUserPolicyMap.Load(accessKey)
-			pols, _ := cache.iamUserPolicyMap.Load(accessKey)
 		for _, policy := range pols.toSlice() {
-				if _, found = cache.iamPolicyDocsMap[policy]; !found {
+			if _, found = newCache.iamPolicyDocsMap[policy]; !found {
-					err = store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadPolicyDocWithRetry(ctx, policy, newCache.iamPolicyDocsMap, 3)
 			}
 		}
+
+		pols, _ = newCache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
+		for _, policy := range pols.toSlice() {
+			if _, found = newCache.iamPolicyDocsMap[policy]; !found {
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadPolicyDocWithRetry(ctx, policy, newCache.iamPolicyDocsMap, 3)
+			}
+		}
+
+		if groupLoad {
+			// NOTE: we are not worried about loading errors from groups.
+			store.updateGroups(ctx, newCache)
+			newCache.buildUserGroupMemberships()
+		}
+
+		return newCache, err
+	}
+
+	var (
+		val interface{}
+		err error
+	)
+	if store.group != nil {
+		val, err, _ = store.group.Do(accessKey, newCachePopulate)
 	} else {
-			pols, _ := cache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
+		val, err = newCachePopulate()
-			for _, policy := range pols.toSlice() {
-				if _, found = cache.iamPolicyDocsMap[policy]; !found {
-					err = store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
-				}
-			}
 	}
 
-		load := len(cache.iamGroupsMap) == 0
+	// Return error right away if any.
-		if store.getUsersSysType() == LDAPUsersSysType && cache.iamGroupPolicyMap.Size() == 0 {
+	if err != nil {
-			load = true
+		if errors.Is(err, errNoSuchUser) || errors.Is(err, errConfigNotFound) {
-		}
-		if load {
-			if _, err = store.updateGroups(ctx, cache); err != nil {
-				return "done", err
-			}
-		}
-
-		cache.buildUserGroupMemberships()
-
-		return "done", err
-	})
-
-	if serverDebugLog {
-		console.Debugln("loadUser: loading shared", val, err, shared)
-	}
-
-	if IsErr(err, errNoSuchUser, errNoSuchPolicy, errNoSuchGroup) {
 			return nil
 		}
-
 		return err
 	}
 
-func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
+	newCache := val.(*iamCache)
-	jwtClaims, err := auth.ExtractClaims(u.Credentials.SessionToken, u.Credentials.SecretKey)
+
-	if err != nil {
+	cache := store.lock()
+	defer store.unlock()
+
+	// We need to merge the new cache with the existing cache because the
+	// periodic IAM reload is partial. The periodic load here is to account.
+	newCache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		cache.iamGroupPolicyMap.Store(k, v)
+		return true
+	})
+
+	for k, v := range newCache.iamGroupsMap {
+		cache.iamGroupsMap[k] = v
+	}
+
+	for k, v := range newCache.iamPolicyDocsMap {
+		cache.iamPolicyDocsMap[k] = v
+	}
+
+	for k, v := range newCache.iamUserGroupMemberships {
+		cache.iamUserGroupMemberships[k] = v
+	}
+
+	newCache.iamUserPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		cache.iamUserPolicyMap.Store(k, v)
+		return true
+	})
+
+	for k, v := range newCache.iamUsersMap {
+		cache.iamUsersMap[k] = v
+	}
+
+	for k, v := range newCache.iamSTSAccountsMap {
+		cache.iamSTSAccountsMap[k] = v
+	}
+
+	newCache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		cache.iamSTSPolicyMap.Store(k, v)
+		return true
+	})
+
+	cache.updatedAt = time.Now()
+
+	return nil
+}
+
+func extractJWTClaims(u UserIdentity) (jwtClaims *jwt.MapClaims, err error) {
+	keys := make([]string, 0, 3)
+
+	// Append credentials secret key itself
+	keys = append(keys, u.Credentials.SecretKey)
+
+	// Use site-replication credentials if found
+	if globalSiteReplicationSys.isEnabled() {
 		secretKey, err := getTokenSigningKey()
 		if err != nil {
 			return nil, err
 		}
-		// Session tokens for STS creds will be generated with root secret or site-replicator-0 secret
+		keys = append(keys, secretKey)
-		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, secretKey)
+	}
-		if err != nil {
+
-			return nil, err
+	// Iterate over all keys and return with the first successful claim extraction
+	for _, key := range keys {
+		jwtClaims, err = getClaimsFromTokenWithSecret(u.Credentials.SessionToken, key)
+		if err == nil {
+			break
 		}
 	}
-	return jwtClaims, nil
+	return
 }
 
 func validateSvcExpirationInUTC(expirationInUTC time.Time) error {

cmd/iam.go

@@ -46,9 +46,9 @@ import (
 	"github.com/minio/minio/internal/config/policy/opa"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	xhttp "github.com/minio/minio/internal/http"
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/ldap"
 	"github.com/minio/pkg/v3/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -178,9 +178,16 @@ func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
 	}
 
 	if etcdClient == nil {
-		sys.store = &IAMStoreSys{newIAMObjectStore(objAPI, sys.usersSysType), &singleflight.Group{}}
+		var group *singleflight.Group
+		if env.Get("_MINIO_IAM_SINGLE_FLIGHT", config.EnableOn) == config.EnableOn {
+			group = &singleflight.Group{}
+		}
+		sys.store = &IAMStoreSys{
+			IAMStorageAPI: newIAMObjectStore(objAPI, sys.usersSysType),
+			group:         group,
+		}
 	} else {
-		sys.store = &IAMStoreSys{newIAMEtcdStore(etcdClient, sys.usersSysType), &singleflight.Group{}}
+		sys.store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(etcdClient, sys.usersSysType)}
 	}
 }
 
@@ -211,7 +218,7 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	if !globalSiteReplicatorCred.IsValid() {
 		sa, _, err := sys.getServiceAccount(ctx, siteReplicatorSvcAcc)
 		if err == nil {
-			globalSiteReplicatorCred.Set(sa.Credentials)
+			globalSiteReplicatorCred.Set(sa.Credentials.SecretKey)
 		}
 	}
 
@@ -225,7 +232,7 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	select {
 	case <-sys.configLoaded:
 	default:
-		xioutil.SafeClose(sys.configLoaded)
+		close(sys.configLoaded)
 	}
 	return nil
 }
@@ -307,8 +314,9 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		// Migrate IAM configuration, if necessary.
 		if err := saveIAMFormat(retryCtx, sys.store); err != nil {
 			if configRetriableErrors(err) {
-				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
+				retryInterval := time.Duration(r.Float64() * float64(time.Second))
-				time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
+				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v) (retrying in %s)", err, retryInterval)
+				time.Sleep(retryInterval)
 				continue
 			}
 			iamLogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err), logger.WarningKind)
@@ -340,8 +348,9 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	for {
 		if err := sys.Load(retryCtx, true); err != nil {
 			if configRetriableErrors(err) {
-				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
+				retryInterval := time.Duration(r.Float64() * float64(time.Second))
-				time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
+				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v) (retrying in %s)", err, retryInterval)
+				time.Sleep(retryInterval)
 				continue
 			}
 			if err != nil {
@@ -397,6 +406,7 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 	timer := time.NewTimer(waitInterval())
 	defer timer.Stop()
 
+	lastPurgeHour := -1
 	for {
 		select {
 		case <-timer.C:
@@ -408,13 +418,13 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 				took := time.Since(refreshStart).Seconds()
 				if took > maxDurationSecondsForLog {
 					// Log if we took a lot of time to load.
-					logger.Info("IAM refresh took (duration: %s)", took)
+					logger.Info("IAM refresh took (duration: %.2fs)", took)
 				}
 			}
 
-			// The following actions are performed about once in 4 times that
+			// Run purge routines once in each hour.
-			// IAM is refreshed:
+			if refreshStart.Hour() != lastPurgeHour {
-			if r.Intn(4) == 0 {
+				lastPurgeHour = refreshStart.Hour()
 				// Poll and remove accounts for those users who were removed
 				// from LDAP/OpenID.
 				if sys.LDAPConfig.Enabled() {
@@ -1212,10 +1222,19 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (Use
 
 // GetTemporaryAccount - wrapper method to get information about a temporary account
 func (sys *IAMSys) GetTemporaryAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) {
+	if !sys.Initialized() {
+		return auth.Credentials{}, nil, errServerNotInitialized
+	}
 	tmpAcc, embeddedPolicy, err := sys.getTempAccount(ctx, accessKey)
+	if err != nil {
+		if err == errNoSuchTempAccount {
+			sys.store.LoadUser(ctx, accessKey)
+			tmpAcc, embeddedPolicy, err = sys.getTempAccount(ctx, accessKey)
+		}
 		if err != nil {
 			return auth.Credentials{}, nil, err
 		}
+	}
 	// Hide secret & session keys
 	tmpAcc.Credentials.SecretKey = ""
 	tmpAcc.Credentials.SessionToken = ""
@@ -1276,10 +1295,6 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errServerNotInitialized
 	}
 
-	if sys.usersSysType != LDAPUsersSysType {
-		return nil, nil
-	}
-
 	sa, ok := sys.store.GetUser(accessKey)
 	if !ok || !sa.Credentials.IsServiceAccount() {
 		return nil, errNoSuchServiceAccount
@@ -1550,16 +1565,16 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 // accounts) for LDAP users. This normalizes the parent user and the group names
 // whenever the parent user parses validly as a DN.
 func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap map[string]madmin.SRSvcAccCreate,
-) (err error) {
+) (skippedAccessKeys []string, err error) {
 	conn, err := sys.LDAPConfig.LDAP.Connect()
 	if err != nil {
-		return err
+		return skippedAccessKeys, err
 	}
 	defer conn.Close()
 
 	// Bind to the lookup user account
 	if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil {
-		return err
+		return skippedAccessKeys, err
 	}
 
 	var collectedErrors []error
@@ -1584,8 +1599,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 			continue
 		}
 		if validatedParent == nil || !isUnderBaseDN {
-			err := fmt.Errorf("DN parent was not found in the LDAP directory")
+			skippedAccessKeys = append(skippedAccessKeys, ak)
-			collectedErrors = append(collectedErrors, err)
 			continue
 		}
 
@@ -1603,8 +1617,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 				continue
 			}
 			if validatedGroup == nil {
-				err := fmt.Errorf("DN group was not found in the LDAP directory")
+				// DN group was not found in the LDAP directory for access-key
-				collectedErrors = append(collectedErrors, err)
 				continue
 			}
 
@@ -1625,7 +1638,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 
 	// if there are any errors, return a collected error.
 	if len(collectedErrors) > 0 {
-		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
+		return skippedAccessKeys, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
 	}
 
 	for k, v := range updatedKeysMap {
@@ -1633,7 +1646,7 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 		accessKeyMap[k] = v
 	}
 
-	return nil
+	return skippedAccessKeys, nil
 }
 
 func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet {
@@ -1659,16 +1672,16 @@ func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup b
 // normalized form.
 func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 	policyMap map[string]MappedPolicy,
-) error {
+) ([]string, error) {
 	conn, err := sys.LDAPConfig.LDAP.Connect()
 	if err != nil {
-		return err
+		return []string{}, err
 	}
 	defer conn.Close()
 
 	// Bind to the lookup user account
 	if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil {
-		return err
+		return []string{}, err
 	}
 
 	// We map keys that correspond to LDAP DNs and validate that they exist in
@@ -1681,6 +1694,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 	// map of normalized DN keys to original keys.
 	normalizedDNKeysMap := make(map[string][]string)
 	var collectedErrors []error
+	var skipped []string
 	for k := range policyMap {
 		_, err := ldap.NormalizeDN(k)
 		if err != nil {
@@ -1693,8 +1707,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			continue
 		}
 		if validatedDN == nil || !underBaseDN {
-			err := fmt.Errorf("DN was not found in the LDAP directory")
+			skipped = append(skipped, k)
-			collectedErrors = append(collectedErrors, err)
 			continue
 		}
 
@@ -1705,7 +1718,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 
 	// if there are any errors, return a collected error.
 	if len(collectedErrors) > 0 {
-		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
+		return []string{}, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
 	}
 
 	entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup)
@@ -1726,7 +1739,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			}
 
 			if policiesDiffer {
-				return fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed",
+				return []string{}, fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed",
 					normKey, origKeys)
 			}
 
@@ -1770,7 +1783,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			}
 		}
 	}
-	return nil
+	return skipped, nil
 }
 
 // CheckKey validates the incoming accessKey
@@ -2163,7 +2176,6 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 			return false
 		}
 		svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
-
 	default:
 		// Check policy for parent user of service account.
 		svcPolicies, err = sys.PolicyDBGet(parentUser, args.Groups...)

cmd/kms-handlers.go

@@ -22,7 +22,6 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/kms"
@@ -197,7 +196,7 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	allKeyNames, _, err := GlobalKMS.ListKeyNames(ctx, &kms.ListRequest{
+	allKeys, _, err := GlobalKMS.ListKeys(ctx, &kms.ListRequest{
 		Prefix: r.Form.Get("pattern"),
 	})
 	if err != nil {
@@ -213,21 +212,17 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	// Now we have all the key names, for each of them, check whether the policy grants permission for
-	// the user to list it.
+	// the user to list it. Filter in place to leave only allowed keys.
-	keyNames := []string{}
+	n := 0
-	for _, name := range allKeyNames {
+	for _, k := range allKeys {
-		if checkKMSActionAllowed(r, owner, cred, policy.KMSListKeysAction, name) {
+		if checkKMSActionAllowed(r, owner, cred, policy.KMSListKeysAction, k.Name) {
-			keyNames = append(keyNames, name)
+			allKeys[n] = k
+			n++
 		}
 	}
+	allKeys = allKeys[:n]
 
-	values := make([]kes.KeyInfo, 0, len(keyNames))
+	if res, err := json.Marshal(allKeys); err != nil {
-	for _, name := range keyNames {
-		values = append(values, kes.KeyInfo{
-			Name: name,
-		})
-	}
-	if res, err := json.Marshal(values); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
 	} else {
 		writeSuccessResponseJSON(w, res)

cmd/kms-handlers_test.go

@@ -658,13 +658,24 @@ func execKMSTest(t *testing.T, test kmsTestCase, adminTestBed *adminErasureTestB
 
 	// Check returned key list is correct
 	if test.wantKeyNames != nil {
-		gotKeyNames := keyNamesFromListKeysResp(t, rec.Body.Bytes())
+		keys := []madmin.KMSKeyInfo{}
-		if len(test.wantKeyNames) != len(gotKeyNames) {
+		err := json.Unmarshal(rec.Body.Bytes(), &keys)
-			t.Fatalf("want keys len: %d, got len: %d", len(test.wantKeyNames), len(gotKeyNames))
+		if err != nil {
+			t.Fatal(err)
 		}
-		for i, wantKeyName := range test.wantKeyNames {
+		if len(keys) != len(test.wantKeyNames) {
-			if gotKeyNames[i] != wantKeyName {
+			t.Fatalf("want %d keys, got %d", len(test.wantKeyNames), len(keys))
-				t.Fatalf("want key name %s, in position %d, got %s", wantKeyName, i, gotKeyNames[i])
+		}
+
+		for i, want := range keys {
+			if want.CreatedBy != kms.StubCreatedBy {
+				t.Fatalf("want key created by %s, got %s", kms.StubCreatedBy, want.CreatedBy)
+			}
+			if want.CreatedAt != kms.StubCreatedAt {
+				t.Fatalf("want key created at %s, got %s", kms.StubCreatedAt, want.CreatedAt)
+			}
+			if test.wantKeyNames[i] != want.Name {
+				t.Fatalf("want key name %s, got %s", test.wantKeyNames[i], want.Name)
 			}
 		}
 	}
@@ -835,17 +846,3 @@ func setupKMSUser(t *testing.T, accessKey, secretKey, p string) {
 		}
 	}
 }
-
-func keyNamesFromListKeysResp(t *testing.T, b []byte) []string {
-	var keyInfos []madmin.KMSKeyInfo
-	err := json.Unmarshal(b, &keyInfos)
-	if err != nil {
-		t.Fatalf("cannot unmarshal '%s', err: %v", b, err)
-	}
-
-	var gotKeyNames []string
-	for _, keyInfo := range keyInfos {
-		gotKeyNames = append(gotKeyNames, keyInfo.Name)
-	}
-	return gotKeyNames
-}

cmd/metacache-entries.go

@@ -257,7 +257,7 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 				ModTime:  timeSentinel1970,
 			}, nil
 		}
-		return e.cached.ToFileInfo(bucket, e.name, "", false, false)
+		return e.cached.ToFileInfo(bucket, e.name, "", false, true)
 	}
 	return getFileInfo(e.metadata, bucket, e.name, "", fileInfoOpts{})
 }
@@ -300,7 +300,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro
 		}, nil
 	}
 	// Too small gains to reuse cache here.
-	return getFileInfoVersions(e.metadata, bucket, e.name, false, true)
+	return getFileInfoVersions(e.metadata, bucket, e.name, true)
 }
 
 // metaCacheEntries is a slice of metacache entries.

cmd/metrics-v2.go

@@ -2313,8 +2313,8 @@ func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
 		var ml []MetricV2
 		// common operational metrics for bucket replication and site replication - published
 		// at cluster level
-		if globalReplicationStats != nil {
+		if rStats := globalReplicationStats.Load(); rStats != nil {
-			qs := globalReplicationStats.getNodeQueueStatsSummary()
+			qs := rStats.getNodeQueueStatsSummary()
 			activeWorkersCount := MetricV2{
 				Description: getClusterReplActiveWorkersCountMD(),
 			}
@@ -3245,7 +3245,7 @@ func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
 
 		var bucketReplStats map[string]BucketStats
 		if !globalSiteReplicationSys.isEnabled() {
-			bucketReplStats = globalReplicationStats.getAllLatest(dataUsageInfo.BucketsUsage)
+			bucketReplStats = globalReplicationStats.Load().getAllLatest(dataUsageInfo.BucketsUsage)
 		}
 		for bucket, usage := range dataUsageInfo.BucketsUsage {
 			quota, _ := globalBucketQuotaSys.Get(ctx, bucket)

cmd/metrics-v3-api.go

@@ -129,7 +129,7 @@ func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsC
 // This is a `MetricsLoaderFn`.
 func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
 	renameLabels := map[string]string{"api": "name"}
-	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	labelsFilter := map[string]set.StringSet{}
 	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil,
 		"type", "s3")
 	return nil
@@ -217,7 +217,7 @@ func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCac
 // This is a `MetricsLoaderFn`.
 func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
 	renameLabels := map[string]string{"api": "name"}
-	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	labelsFilter := map[string]set.StringSet{}
 	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels,
 		buckets, "type", "s3")
 	return nil

cmd/metrics-v3-bucket-replication.go

@@ -119,7 +119,7 @@ func loadBucketReplicationMetrics(ctx context.Context, m MetricValues, c *metric
 		return nil
 	}
 
-	bucketReplStats := globalReplicationStats.getAllLatest(dataUsageInfo.BucketsUsage)
+	bucketReplStats := globalReplicationStats.Load().getAllLatest(dataUsageInfo.BucketsUsage)
 	for _, bucket := range buckets {
 		labels := []string{bucketL, bucket}
 		if s, ok := bucketReplStats[bucket]; ok {

cmd/metrics-v3-replication.go

@@ -69,11 +69,12 @@ var (
 // loadClusterReplicationMetrics - `MetricsLoaderFn` for cluster replication metrics
 // such as transfer rate and objects queued.
 func loadClusterReplicationMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
-	if globalReplicationStats == nil {
+	st := globalReplicationStats.Load()
+	if st == nil {
 		return nil
 	}
 
-	qs := globalReplicationStats.getNodeQueueStatsSummary()
+	qs := st.getNodeQueueStatsSummary()
 
 	qt := qs.QStats
 	m.Set(replicationAverageQueuedBytes, float64(qt.Avg.Bytes))

cmd/metrics.go

@@ -300,7 +300,7 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	}
 
 	for bucket, usageInfo := range dataUsageInfo.BucketsUsage {
-		stat := globalReplicationStats.getLatestReplicationStats(bucket)
+		stat := globalReplicationStats.Load().getLatestReplicationStats(bucket)
 		// Total space used by bucket
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(

cmd/mrf.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -15,51 +15,203 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:generate msgp -file=$GOFILE
+
 package cmd
 
 import (
 	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/pkg/v3/wildcard"
+	"github.com/tinylib/msgp/msgp"
 )
 
 const (
 	mrfOpsQueueSize = 100000
 )
 
-// partialOperation is a successful upload/delete of an object
+const (
+	healDir              = ".heal"
+	healMRFDir           = bucketMetaPrefix + SlashSeparator + healDir + SlashSeparator + "mrf"
+	healMRFMetaFormat    = 1
+	healMRFMetaVersionV1 = 1
+)
+
+// PartialOperation is a successful upload/delete of an object
 // but not written in all disks (having quorum)
-type partialOperation struct {
+type PartialOperation struct {
-	bucket              string
+	Bucket              string
-	object              string
+	Object              string
-	versionID           string
+	VersionID           string
-	versions            []byte
+	Versions            []byte
-	setIndex, poolIndex int
+	SetIndex, PoolIndex int
-	queued              time.Time
+	Queued              time.Time
-	scanMode            madmin.HealScanMode
+	BitrotScan          bool
 }
 
 // mrfState sncapsulates all the information
 // related to the global background MRF.
 type mrfState struct {
-	opCh chan partialOperation
+	opCh chan PartialOperation
+
+	closed  int32
+	closing int32
+	wg      sync.WaitGroup
+}
+
+func newMRFState() mrfState {
+	return mrfState{
+		opCh: make(chan PartialOperation, mrfOpsQueueSize),
+	}
 }
 
 // Add a partial S3 operation (put/delete) when one or more disks are offline.
-func (m *mrfState) addPartialOp(op partialOperation) {
+func (m *mrfState) addPartialOp(op PartialOperation) {
 	if m == nil {
 		return
 	}
 
+	if atomic.LoadInt32(&m.closed) == 1 {
+		return
+	}
+
+	m.wg.Add(1)
+	defer m.wg.Done()
+
+	if atomic.LoadInt32(&m.closing) == 1 {
+		return
+	}
+
 	select {
 	case m.opCh <- op:
 	default:
 	}
 }
 
+// Do not accept new MRF operations anymore and start to save
+// the current heal status in one available disk
+func (m *mrfState) shutdown() {
+	atomic.StoreInt32(&m.closing, 1)
+	m.wg.Wait()
+	close(m.opCh)
+	atomic.StoreInt32(&m.closed, 1)
+
+	if len(m.opCh) > 0 {
+		healingLogEvent(context.Background(), "Saving MRF healing data (%d entries)", len(m.opCh))
+	}
+
+	newReader := func() io.ReadCloser {
+		r, w := io.Pipe()
+		go func() {
+			// Initialize MRF meta header.
+			var data [4]byte
+			binary.LittleEndian.PutUint16(data[0:2], healMRFMetaFormat)
+			binary.LittleEndian.PutUint16(data[2:4], healMRFMetaVersionV1)
+			mw := msgp.NewWriter(w)
+			n, err := mw.Write(data[:])
+			if err != nil {
+				w.CloseWithError(err)
+				return
+			}
+			if n != len(data) {
+				w.CloseWithError(io.ErrShortWrite)
+				return
+			}
+			for item := range m.opCh {
+				err = item.EncodeMsg(mw)
+				if err != nil {
+					break
+				}
+			}
+			mw.Flush()
+			w.CloseWithError(err)
+		}()
+		return r
+	}
+
+	globalLocalDrivesMu.RLock()
+	localDrives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+
+	for _, localDrive := range localDrives {
+		r := newReader()
+		err := localDrive.CreateFile(context.Background(), "", minioMetaBucket, pathJoin(healMRFDir, "list.bin"), -1, r)
+		r.Close()
+		if err == nil {
+			break
+		}
+	}
+}
+
+func (m *mrfState) startMRFPersistence() {
+	loadMRF := func(rc io.ReadCloser, opCh chan PartialOperation) error {
+		defer rc.Close()
+		var data [4]byte
+		n, err := rc.Read(data[:])
+		if err != nil {
+			return err
+		}
+		if n != len(data) {
+			return errors.New("heal mrf: no data")
+		}
+		// Read resync meta header
+		switch binary.LittleEndian.Uint16(data[0:2]) {
+		case healMRFMetaFormat:
+		default:
+			return fmt.Errorf("heal mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		}
+		switch binary.LittleEndian.Uint16(data[2:4]) {
+		case healMRFMetaVersionV1:
+		default:
+			return fmt.Errorf("heal mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		}
+
+		mr := msgp.NewReader(rc)
+		for {
+			op := PartialOperation{}
+			err = op.DecodeMsg(mr)
+			if err != nil {
+				break
+			}
+			opCh <- op
+		}
+
+		return nil
+	}
+
+	globalLocalDrivesMu.RLock()
+	localDrives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+
+	for _, localDrive := range localDrives {
+		if localDrive == nil {
+			continue
+		}
+		rc, err := localDrive.ReadFileStream(context.Background(), minioMetaBucket, pathJoin(healMRFDir, "list.bin"), 0, -1)
+		if err != nil {
+			continue
+		}
+		err = loadMRF(rc, m.opCh)
+		if err != nil {
+			continue
+		}
+		// finally delete the file after processing mrf entries
+		localDrive.Delete(GlobalContext, minioMetaBucket, pathJoin(healMRFDir, "list.bin"), DeleteOptions{})
+		break
+	}
+
+	return
+}
+
 var healSleeper = newDynamicSleeper(5, time.Second, false)
 
 // healRoutine listens to new disks reconnection events and
@@ -78,24 +230,24 @@ func (m *mrfState) healRoutine(z *erasureServerPools) {
 			// We might land at .metacache, .trash, .multipart
 			// no need to heal them skip, only when bucket
 			// is '.minio.sys'
-			if u.bucket == minioMetaBucket {
+			if u.Bucket == minioMetaBucket {
 				// No MRF needed for temporary objects
-				if wildcard.Match("buckets/*/.metacache/*", u.object) {
+				if wildcard.Match("buckets/*/.metacache/*", u.Object) {
 					continue
 				}
-				if wildcard.Match("tmp/*", u.object) {
+				if wildcard.Match("tmp/*", u.Object) {
 					continue
 				}
-				if wildcard.Match("multipart/*", u.object) {
+				if wildcard.Match("multipart/*", u.Object) {
 					continue
 				}
-				if wildcard.Match("tmp-old/*", u.object) {
+				if wildcard.Match("tmp-old/*", u.Object) {
 					continue
 				}
 			}
 
 			now := time.Now()
-			if now.Sub(u.queued) < time.Second {
+			if now.Sub(u.Queued) < time.Second {
 				// let recently failed networks to reconnect
 				// making MRF wait for 1s before retrying,
 				// i.e 4 reconnect attempts.
@@ -106,21 +258,22 @@ func (m *mrfState) healRoutine(z *erasureServerPools) {
 			wait := healSleeper.Timer(context.Background())
 
 			scan := madmin.HealNormalScan
-			if u.scanMode != 0 {
+			if u.BitrotScan {
-				scan = u.scanMode
+				scan = madmin.HealDeepScan
 			}
-			if u.object == "" {
+
-				healBucket(u.bucket, scan)
+			if u.Object == "" {
+				healBucket(u.Bucket, scan)
 			} else {
-				if len(u.versions) > 0 {
+				if len(u.Versions) > 0 {
-					vers := len(u.versions) / 16
+					vers := len(u.Versions) / 16
 					if vers > 0 {
 						for i := 0; i < vers; i++ {
-							healObject(u.bucket, u.object, uuid.UUID(u.versions[16*i:]).String(), scan)
+							healObject(u.Bucket, u.Object, uuid.UUID(u.Versions[16*i:]).String(), scan)
 						}
 					}
 				} else {
-					healObject(u.bucket, u.object, u.versionID, scan)
+					healObject(u.Bucket, u.Object, u.VersionID, scan)
 				}
 			}
 

cmd/mrf_gen.go

@@ -0,0 +1,285 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *PartialOperation) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Object":
+			z.Object, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		case "VersionID":
+			z.VersionID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "VersionID")
+				return
+			}
+		case "Versions":
+			z.Versions, err = dc.ReadBytes(z.Versions)
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "SetIndex":
+			z.SetIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "SetIndex")
+				return
+			}
+		case "PoolIndex":
+			z.PoolIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "PoolIndex")
+				return
+			}
+		case "Queued":
+			z.Queued, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Queued")
+				return
+			}
+		case "BitrotScan":
+			z.BitrotScan, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "BitrotScan")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *PartialOperation) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 8
+	// write "Bucket"
+	err = en.Append(0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Object"
+	err = en.Append(0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Object)
+	if err != nil {
+		err = msgp.WrapError(err, "Object")
+		return
+	}
+	// write "VersionID"
+	err = en.Append(0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.VersionID)
+	if err != nil {
+		err = msgp.WrapError(err, "VersionID")
+		return
+	}
+	// write "Versions"
+	err = en.Append(0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.Versions)
+	if err != nil {
+		err = msgp.WrapError(err, "Versions")
+		return
+	}
+	// write "SetIndex"
+	err = en.Append(0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.SetIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "SetIndex")
+		return
+	}
+	// write "PoolIndex"
+	err = en.Append(0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.PoolIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "PoolIndex")
+		return
+	}
+	// write "Queued"
+	err = en.Append(0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Queued)
+	if err != nil {
+		err = msgp.WrapError(err, "Queued")
+		return
+	}
+	// write "BitrotScan"
+	err = en.Append(0xaa, 0x42, 0x69, 0x74, 0x72, 0x6f, 0x74, 0x53, 0x63, 0x61, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.BitrotScan)
+	if err != nil {
+		err = msgp.WrapError(err, "BitrotScan")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *PartialOperation) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 8
+	// string "Bucket"
+	o = append(o, 0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Object"
+	o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
+	o = msgp.AppendString(o, z.Object)
+	// string "VersionID"
+	o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44)
+	o = msgp.AppendString(o, z.VersionID)
+	// string "Versions"
+	o = append(o, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	o = msgp.AppendBytes(o, z.Versions)
+	// string "SetIndex"
+	o = append(o, 0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.SetIndex)
+	// string "PoolIndex"
+	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.PoolIndex)
+	// string "Queued"
+	o = append(o, 0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.Queued)
+	// string "BitrotScan"
+	o = append(o, 0xaa, 0x42, 0x69, 0x74, 0x72, 0x6f, 0x74, 0x53, 0x63, 0x61, 0x6e)
+	o = msgp.AppendBool(o, z.BitrotScan)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *PartialOperation) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Object":
+			z.Object, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		case "VersionID":
+			z.VersionID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersionID")
+				return
+			}
+		case "Versions":
+			z.Versions, bts, err = msgp.ReadBytesBytes(bts, z.Versions)
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "SetIndex":
+			z.SetIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "SetIndex")
+				return
+			}
+		case "PoolIndex":
+			z.PoolIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PoolIndex")
+				return
+			}
+		case "Queued":
+			z.Queued, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Queued")
+				return
+			}
+		case "BitrotScan":
+			z.BitrotScan, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BitrotScan")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *PartialOperation) Msgsize() (s int) {
+	s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 10 + msgp.StringPrefixSize + len(z.VersionID) + 9 + msgp.BytesPrefixSize + len(z.Versions) + 9 + msgp.IntSize + 10 + msgp.IntSize + 7 + msgp.TimeSize + 11 + msgp.BoolSize
+	return
+}

cmd/mrf_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalPartialOperation(t *testing.T) {
+	v := PartialOperation{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgPartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgPartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalPartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodePartialOperation(t *testing.T) {
+	v := PartialOperation{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodePartialOperation Msgsize() is inaccurate")
+	}
+
+	vn := PartialOperation{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodePartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodePartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/namespace-lock.go

@@ -185,7 +185,7 @@ func (di *distLockInstance) Unlock(lc LockContext) {
 	if lc.cancel != nil {
 		lc.cancel()
 	}
-	di.rwMutex.Unlock(lc.ctx)
+	di.rwMutex.Unlock(context.Background())
 }
 
 // RLock - block until read lock is taken or timeout has occurred.
@@ -229,6 +229,7 @@ type localLockInstance struct {
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
 func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume string, paths ...string) RWLocker {
+	sort.Strings(paths)
 	opsID := mustGetUUID()
 	if n.isDistErasure {
 		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
@@ -237,7 +238,6 @@ func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume
 		}, pathsJoinPrefix(volume, paths...)...)
 		return &distLockInstance{drwmutex, opsID}
 	}
-	sort.Strings(paths)
 	return &localLockInstance{n, volume, paths, opsID}
 }
 

cmd/naughty-disk_test.go

@@ -208,6 +208,20 @@ func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath string,
 	return d.disk.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
 }
 
+func (d *naughtyDisk) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta)
+}
+
+func (d *naughtyDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.ReadParts(ctx, bucket, partMetaPaths...)
+}
+
 func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
 	if err := d.calcError(); err != nil {
 		return err

cmd/notification.go

@@ -84,6 +84,9 @@ func WithNPeersThrottled(nerrs, wks int) *NotificationGroup {
 	if nerrs <= 0 {
 		nerrs = 1
 	}
+	if wks > nerrs {
+		wks = nerrs
+	}
 	wk, _ := workers.New(wks)
 	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), workers: wk, retryCount: 3}
 }
@@ -292,15 +295,15 @@ func (sys *NotificationSys) BackgroundHealStatus(ctx context.Context) ([]madmin.
 }
 
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.
-func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
+func (sys *NotificationSys) StartProfiling(ctx context.Context, profiler string) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
 		client := client
-		ng.Go(GlobalContext, func() error {
+		ng.Go(ctx, func() error {
-			return client.StartProfiling(profiler)
+			return client.StartProfiling(ctx, profiler)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -313,28 +316,49 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 	zipWriter := zip.NewWriter(writer)
 	defer zipWriter.Close()
 
-	for _, client := range sys.peerClients {
+	// Start by embedding cluster info.
+	if b := getClusterMetaInfo(ctx); len(b) > 0 {
+		internalLogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b, 0o600))
+	}
+
+	// Profiles can be quite big, so we limit to max 16 concurrent downloads.
+	ng := WithNPeersThrottled(len(sys.peerClients), 16)
+	var writeMu sync.Mutex
+	for i, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
-		data, err := client.DownloadProfileData()
+		ng.Go(ctx, func() error {
+			// Give 15 seconds to each remote call.
+			// Errors are logged but not returned.
+			ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+			defer cancel()
+			data, err := client.DownloadProfileData(ctx)
 			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 				ctx := logger.SetReqInfo(ctx, reqInfo)
 				peersLogOnceIf(ctx, err, client.host.String())
-			continue
+				return nil
 			}
 
-		profilingDataFound = true
-
 			for typ, data := range data {
+				// zip writer only handles one concurrent write
+				writeMu.Lock()
+				profilingDataFound = true
 				err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data, 0o600)
+				writeMu.Unlock()
 				if err != nil {
 					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 					ctx := logger.SetReqInfo(ctx, reqInfo)
 					peersLogOnceIf(ctx, err, client.host.String())
 				}
 			}
+			return nil
+		}, i, *client.host)
+	}
+	ng.Wait()
+	if ctx.Err() != nil {
+		return false
 	}
 
 	// Local host
@@ -359,9 +383,6 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data, 0o600)
 		internalLogIf(ctx, err)
 	}
-	if b := getClusterMetaInfo(ctx); len(b) > 0 {
-		internalLogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b, 0o600))
-	}
 
 	return
 }
@@ -383,10 +404,6 @@ func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256
 	// further discussion advised. Remove this comment and remove the worker model
 	// for this function in future.
 	maxWorkers := runtime.GOMAXPROCS(0) / 2
-	if maxWorkers > len(sys.peerClients) {
-		maxWorkers = len(sys.peerClients)
-	}
-
 	ng := WithNPeersThrottled(len(sys.peerClients), maxWorkers)
 	for idx, client := range sys.peerClients {
 		if client == nil {
@@ -520,7 +537,7 @@ func (sys *NotificationSys) LoadBucketMetadata(ctx context.Context, bucketName s
 
 // DeleteBucketMetadata - calls DeleteBucketMetadata call on all peers
 func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName string) {
-	globalReplicationStats.Delete(bucketName)
+	globalReplicationStats.Load().Delete(bucketName)
 	globalBucketMetadataSys.Remove(bucketName)
 	globalBucketTargetSys.Delete(bucketName)
 	globalEventNotifier.RemoveNotification(bucketName)
@@ -574,7 +591,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
 		}
 	}
 
-	replicationStatsList := globalReplicationStats.GetAll()
+	replicationStatsList := globalReplicationStats.Load().GetAll()
 	bucketStatsMap := BucketStatsMap{
 		Stats:     make(map[string]BucketStats, len(replicationStatsList)),
 		Timestamp: UTCNow(),
@@ -582,7 +599,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
 	for k, replicationStats := range replicationStatsList {
 		bucketStatsMap.Stats[k] = BucketStats{
 			ReplicationStats: replicationStats,
-			ProxyStats:       globalReplicationStats.getProxyStats(k),
+			ProxyStats:       globalReplicationStats.Load().getProxyStats(k),
 		}
 	}
 
@@ -615,11 +632,13 @@ func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketNam
 			peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
+	if st := globalReplicationStats.Load(); st != nil {
 		bucketStats = append(bucketStats, BucketStats{
-		ReplicationStats: globalReplicationStats.Get(bucketName),
+			ReplicationStats: st.Get(bucketName),
-		QueueStats:       ReplicationQueueStats{Nodes: []ReplQNodeStats{globalReplicationStats.getNodeQueueStats(bucketName)}},
+			QueueStats:       ReplicationQueueStats{Nodes: []ReplQNodeStats{st.getNodeQueueStats(bucketName)}},
-		ProxyStats:       globalReplicationStats.getProxyStats(bucketName),
+			ProxyStats:       st.getProxyStats(bucketName),
 		})
+	}
 	return bucketStats
 }
 
@@ -648,7 +667,7 @@ func (sys *NotificationSys) GetClusterSiteMetrics(ctx context.Context) []SRMetri
 			peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
-	siteStats = append(siteStats, globalReplicationStats.getSRMetricsForNode())
+	siteStats = append(siteStats, globalReplicationStats.Load().getSRMetricsForNode())
 	return siteStats
 }
 
@@ -672,6 +691,27 @@ func (sys *NotificationSys) ReloadPoolMeta(ctx context.Context) {
 	}
 }
 
+// DeleteUploadID notifies all the MinIO nodes to remove the
+// given uploadID from cache
+func (sys *NotificationSys) DeleteUploadID(ctx context.Context, uploadID string) {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(ctx, func() error {
+			return client.DeleteUploadID(ctx, uploadID)
+		}, idx, *client.host)
+	}
+	for _, nErr := range ng.Wait() {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
+		if nErr.Err != nil {
+			peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
+		}
+	}
+}
+
 // StopRebalance notifies all MinIO nodes to signal any ongoing rebalance
 // goroutine to stop.
 func (sys *NotificationSys) StopRebalance(ctx context.Context) {
@@ -1099,6 +1139,8 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context, metrics bool) []madm
 		wg.Add(1)
 		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
+			ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+			defer cancel()
 			info, err := client.ServerInfo(ctx, metrics)
 			if err != nil {
 				info.Endpoint = client.host.String()
@@ -1586,7 +1628,7 @@ func (sys *NotificationSys) GetReplicationMRF(ctx context.Context, bucket, node
 		if node != "all" && node != globalLocalNodeName {
 			return nil
 		}
-		mCh, err := globalReplicationPool.getMRF(ctx, bucket)
+		mCh, err := globalReplicationPool.Get().getMRF(ctx, bucket)
 		if err != nil {
 			return err
 		}

cmd/object-api-deleteobject_test.go

@@ -93,14 +93,18 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			md5Bytes := md5.Sum([]byte(object.content))
 			oi, err := obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetPutObjReader(t, strings.NewReader(object.content),
 				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{})
-			t.Log(oi)
 			if err != nil {
+				t.Log(oi)
 				t.Fatalf("%s : %s", instanceType, err.Error())
 			}
 		}
 
 		oi, err := obj.DeleteObject(context.Background(), testCase.bucketName, testCase.pathToDelete, ObjectOptions{})
-		t.Log(oi, err)
+		if err != nil && !isErrObjectNotFound(err) {
+			t.Log(oi)
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err)
+			continue
+		}
 
 		result, err := obj.ListObjects(context.Background(), testCase.bucketName, "", "", "", 1000)
 		if err != nil {

cmd/object-api-errors.go

@@ -788,3 +788,20 @@ func isReplicationPermissionCheck(err error) bool {
 	_, ok := err.(ReplicationPermissionCheck)
 	return ok
 }
+
+// DataMovementOverwriteErr - captures the error when a data movement activity
+// like rebalance incorrectly tries to overwrite an object.
+type DataMovementOverwriteErr GenericError
+
+func (de DataMovementOverwriteErr) Error() string {
+	objInfoStr := fmt.Sprintf("bucket=%s object=%s", de.Bucket, de.Object)
+	if de.VersionID != "" {
+		objInfoStr = fmt.Sprintf("%s version-id=%s", objInfoStr, de.VersionID)
+	}
+	return fmt.Sprintf("invalid data movement operation, source and destination pool are the same for %s", objInfoStr)
+}
+
+func isDataMovementOverWriteErr(err error) bool {
+	var de DataMovementOverwriteErr
+	return errors.As(err, &de)
+}

cmd/object-api-interface.go

@@ -113,6 +113,8 @@ type ObjectOptions struct {
 	// participating in a rebalance operation. Typically set for 'write' operations.
 	SkipRebalancing bool
 
+	SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same
+
 	DataMovement bool // indicates an going decommisionning or rebalacing
 
 	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
@@ -143,6 +145,7 @@ type WalkOptions struct {
 	LatestOnly   bool                     // returns only latest versions for all matching objects
 	AskDisks     string                   // dictates how many disks are being listed
 	VersionsSort WalkVersionsSortOrder    // sort order for versions of the same object; default: Ascending order in ModTime
+	Limit        int                      // maximum number of items, 0 means no limit
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/object-api-interface_gen.go

@@ -219,9 +219,9 @@ func (z *MakeBucketOptions) Msgsize() (s int) {
 // MarshalMsg implements msgp.Marshaler
 func (z *WalkOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
+	// map header, size 5
 	// string "Marker"
-	o = append(o, 0x84, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72)
+	o = append(o, 0x85, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72)
 	o = msgp.AppendString(o, z.Marker)
 	// string "LatestOnly"
 	o = append(o, 0xaa, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79)
@@ -232,6 +232,9 @@ func (z *WalkOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "VersionsSort"
 	o = append(o, 0xac, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x6f, 0x72, 0x74)
 	o = msgp.AppendUint8(o, uint8(z.VersionsSort))
+	// string "Limit"
+	o = append(o, 0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74)
+	o = msgp.AppendInt(o, z.Limit)
 	return
 }
 
@@ -281,6 +284,12 @@ func (z *WalkOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 				z.VersionsSort = WalkVersionsSortOrder(zb0002)
 			}
+		case "Limit":
+			z.Limit, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Limit")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -295,7 +304,7 @@ func (z *WalkOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *WalkOptions) Msgsize() (s int) {
-	s = 1 + 7 + msgp.StringPrefixSize + len(z.Marker) + 11 + msgp.BoolSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 13 + msgp.Uint8Size
+	s = 1 + 7 + msgp.StringPrefixSize + len(z.Marker) + 11 + msgp.BoolSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 13 + msgp.Uint8Size + 6 + msgp.IntSize
 	return
 }
 

cmd/object-api-multipart_test.go

@@ -79,7 +79,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 		case InvalidUploadID:
 			t.Fatalf("%s: New Multipart upload failed to create uuid file.", instanceType)
 		default:
-			t.Fatalf(err.Error())
+			t.Fatal(err.Error())
 		}
 	}
 }
@@ -1202,6 +1202,292 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	}
 }
 
+// Wrapper for calling TestListObjectPartsStale tests for both Erasure multiple disks and single node setup.
+func TestListObjectPartsStale(t *testing.T) {
+	ExecObjectLayerDiskAlteredTest(t, testListObjectPartsStale)
+}
+
+// testListObjectPartsStale - Tests validate listing of object parts when parts are stale
+func testListObjectPartsStale(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
+	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
+	objectNames := []string{"minio-object-1.txt"}
+	uploadIDs := []string{}
+
+	globalStorageClass.Update(storageclass.Config{
+		RRS: storageclass.StorageClass{
+			Parity: 2,
+		},
+		Standard: storageclass.StorageClass{
+			Parity: 4,
+		},
+	})
+
+	// bucketnames[0].
+	// objectNames[0].
+	// uploadIds [0].
+	// Create bucket before initiating NewMultipartUpload.
+	err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{})
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+	opts := ObjectOptions{}
+	// Initiate Multipart Upload on the above created bucket.
+	res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	z := obj.(*erasureServerPools)
+	er := z.serverPools[0].sets[0]
+
+	uploadIDs = append(uploadIDs, res.UploadID)
+
+	// Create multipart parts.
+	// Need parts to be uploaded before MultipartLists can be called and tested.
+	createPartCases := []struct {
+		bucketName      string
+		objName         string
+		uploadID        string
+		PartID          int
+		inputReaderData string
+		inputMd5        string
+		inputDataSize   int64
+		expectedMd5     string
+	}{
+		// Case 1-4.
+		// Creating sequence of parts for same uploadID.
+		// Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"},
+		{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"},
+	}
+	sha256sum := ""
+	// Iterating over creatPartCases to generate multipart chunks.
+	for _, testCase := range createPartCases {
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+
+	erasureDisks := er.getDisks()
+	uploadIDPath := er.getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0])
+	dataDirs, err := erasureDisks[0].ListDir(context.Background(), minioMetaMultipartBucket, minioMetaMultipartBucket, uploadIDPath, -1)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	var dataDir string
+	for _, folder := range dataDirs {
+		if strings.HasSuffix(folder, SlashSeparator) {
+			dataDir = folder
+			break
+		}
+	}
+
+	toDel := (len(erasureDisks) / 2) + 1
+	for _, disk := range erasureDisks[:toDel] {
+		disk.DeleteBulk(context.Background(), minioMetaMultipartBucket, []string{pathJoin(uploadIDPath, dataDir, "part.2")}...)
+	}
+
+	partInfos := []ListPartsInfo{
+		// partinfos - 0.
+		{
+			Bucket:   bucketNames[0],
+			Object:   objectNames[0],
+			MaxParts: 10,
+			UploadID: uploadIDs[0],
+			Parts: []PartInfo{
+				{
+					PartNumber: 1,
+					Size:       4,
+					ETag:       "e2fc714c4727ee9395f324cd2e7f331f",
+				},
+				{
+					PartNumber: 3,
+					Size:       4,
+					ETag:       "09a0877d04abf8759f99adec02baf579",
+				},
+				{
+					PartNumber: 4,
+					Size:       4,
+					ETag:       "e132e96a5ddad6da8b07bba6f6131fef",
+				},
+			},
+		},
+		// partinfos - 1.
+		{
+			Bucket:   bucketNames[0],
+			Object:   objectNames[0],
+			MaxParts: 3,
+			UploadID: uploadIDs[0],
+			Parts: []PartInfo{
+				{
+					PartNumber: 1,
+					Size:       4,
+					ETag:       "e2fc714c4727ee9395f324cd2e7f331f",
+				},
+				{
+					PartNumber: 3,
+					Size:       4,
+					ETag:       "09a0877d04abf8759f99adec02baf579",
+				},
+				{
+					PartNumber: 4,
+					Size:       4,
+					ETag:       "e132e96a5ddad6da8b07bba6f6131fef",
+				},
+			},
+		},
+		// partinfos - 2.
+		{
+			Bucket:               bucketNames[0],
+			Object:               objectNames[0],
+			MaxParts:             2,
+			NextPartNumberMarker: 3,
+			IsTruncated:          true,
+			UploadID:             uploadIDs[0],
+			Parts: []PartInfo{
+				{
+					PartNumber: 1,
+					Size:       4,
+					ETag:       "e2fc714c4727ee9395f324cd2e7f331f",
+				},
+				{
+					PartNumber: 3,
+					Size:       4,
+					ETag:       "09a0877d04abf8759f99adec02baf579",
+				},
+			},
+		},
+		// partinfos - 3.
+		{
+			Bucket:           bucketNames[0],
+			Object:           objectNames[0],
+			MaxParts:         2,
+			IsTruncated:      false,
+			UploadID:         uploadIDs[0],
+			PartNumberMarker: 3,
+			Parts: []PartInfo{
+				{
+					PartNumber: 4,
+					Size:       4,
+					ETag:       "e132e96a5ddad6da8b07bba6f6131fef",
+				},
+			},
+		},
+	}
+
+	// Collection of non-exhaustive ListObjectParts test cases, valid errors
+	// and success responses.
+	testCases := []struct {
+		bucket           string
+		object           string
+		uploadID         string
+		partNumberMarker int
+		maxParts         int
+		// Expected output of ListPartsInfo.
+		expectedResult ListPartsInfo
+		expectedErr    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names (Test number 1-4).
+		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases for listing uploadID with single part.
+		// Valid bucket names, but they do not exist (Test number 5-7).
+		{"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Test case for Asserting for invalid objectName (Test number 8).
+		{bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
+		// Asserting for Invalid UploadID (Test number 9).
+		{bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false},
+		// Test case for uploadID with multiple parts (Test number 12).
+		{bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true},
+		// Test case with maxParts set to less than number of parts (Test number 13).
+		{bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true},
+		// Test case with partNumberMarker set (Test number 14)-.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 0, 2, partInfos[2], nil, true},
+		// Test case with partNumberMarker set (Test number 15)-.
+		{bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[3], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, ObjectOptions{})
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) {
+				t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr, actualErr)
+			}
+		}
+		// Passes as expected, but asserting the results.
+		if actualErr == nil && testCase.shouldPass {
+			expectedResult := testCase.expectedResult
+			// Asserting the MaxParts.
+			if actualResult.MaxParts != expectedResult.MaxParts {
+				t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts)
+			}
+			// Asserting Object Name.
+			if actualResult.Object != expectedResult.Object {
+				t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object)
+			}
+			// Asserting UploadID.
+			if actualResult.UploadID != expectedResult.UploadID {
+				t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID)
+			}
+			// Asserting NextPartNumberMarker.
+			if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
+				t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
+			}
+			// Asserting PartNumberMarker.
+			if actualResult.PartNumberMarker != expectedResult.PartNumberMarker {
+				t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker)
+			}
+			// Asserting the BucketName.
+			if actualResult.Bucket != expectedResult.Bucket {
+				t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
+			}
+			// Asserting IsTruncated.
+			if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
+				t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
+			}
+			// Asserting the number of Parts.
+			if len(expectedResult.Parts) != len(actualResult.Parts) {
+				t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
+			} else {
+				// Iterating over the partInfos and asserting the fields.
+				for j, actualMetaData := range actualResult.Parts {
+					//  Asserting the PartNumber in the PartInfo.
+					if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
+						t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
+					}
+					//  Asserting the Size in the PartInfo.
+					if actualMetaData.Size != expectedResult.Parts[j].Size {
+						t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
+					}
+					//  Asserting the ETag in the PartInfo.
+					if actualMetaData.ETag != expectedResult.Parts[j].ETag {
+						t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
+					}
+				}
+			}
+		}
+	}
+}
+
 // Wrapper for calling TestListObjectPartsDiskNotFound tests for both Erasure multiple disks and single node setup.
 func TestListObjectPartsDiskNotFound(t *testing.T) {
 	ExecObjectLayerDiskAlteredTest(t, testListObjectPartsDiskNotFound)