heal: Include more use case of not healable but readable objects (#248) (#20776)

If one object has many parts where all parts are readable but some parts
are missing from some drives, this object can be sometimes un-healable,
which is wrong.

This commit will avoid reading from drives that have missing, corrupted or
outdated xl.meta. It will also check if any part is unreadable to avoid
healing in that case.

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4

.github/workflows/go-fips.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4

.github/workflows/go-lint.yml

@@ -20,24 +20,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
-        os: [ubuntu-latest, Windows]
+        go-version: [1.23.x]
+        os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - name: Build on ${{ matrix.os }}
-        if: matrix.os == 'Windows'
-        env:
-          CGO_ENABLED: 0
-          GO111MODULE: on
-        run: |
-          Set-MpPreference -DisableRealtimeMonitoring $true
-          netsh int ipv4 set dynamicport tcp start=60000 num=61000
-          go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
-          go test -v --timeout 120m ./...
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
@@ -39,3 +39,4 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify
+          make test-timeout

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -142,3 +142,20 @@ jobs:
       - name: Test import of IAM artifacts when in fresh cluster there are missing groups etc
         run: |
           make test-iam-import-with-missing-entities
+  iam-import-with-openid:
+    name: Test IAM import in new cluster with opendid configurations
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Checkout minio-iam-testing
+        uses: actions/checkout@v4
+        with:
+          repository: minio/minio-iam-testing
+          path: minio-iam-testing
+      - name: Test import of IAM artifacts when in fresh cluster with openid configurations
+        run: |
+          make test-iam-import-with-openid

.github/workflows/mint.yml

@@ -29,7 +29,7 @@ jobs:
       - name: setup-go-step
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.23.x
 
       - name: github sha short
         id: vars

.github/workflows/replication.yaml

@@ -21,7 +21,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
 
     steps:
       - uses: actions/checkout@v4
@@ -40,6 +40,7 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-ilm
+          make test-ilm-transition
 
       - name: Test PBAC
         run: |

.github/workflows/root-disable.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/run-mint.sh

@@ -7,6 +7,7 @@ export ACCESS_KEY="$2"
 export SECRET_KEY="$3"
 export JOB_NAME="$4"
 export MINT_MODE="full"
+export MINT_NO_FULL_OBJECT="true"
 
 docker system prune -f || true
 docker volume prune -f || true
@@ -15,6 +16,9 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 ## change working directory
 cd .github/workflows/mint
 
+## always pull latest
+docker pull docker.io/minio/mint:edge
+
 docker-compose -f minio-${MODE}.yaml up -d
 sleep 1m
 
@@ -35,6 +39,7 @@ docker run --rm --net=mint_default \
 	-e ACCESS_KEY="${ACCESS_KEY}" \
 	-e SECRET_KEY="${SECRET_KEY}" \
 	-e ENABLE_HTTPS=0 \
+	-e MINT_NO_FULL_OBJECT="${MINT_NO_FULL_OBJECT}" \
 	-e MINT_MODE="${MINT_MODE}" \
 	docker.io/minio/mint:edge
 

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.22.x]
+        go-version: [1.23.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/vulncheck.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.7
+          go-version: 1.23.2
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
         shell: bash

Dockerfile

@@ -1,6 +1,6 @@
 FROM minio/minio:latest
 
-RUN chmod 1777 /usr/bin
+RUN chmod -R 777 /usr/bin
 
 COPY ./minio /usr/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

Dockerfile.hotfix

@@ -1,4 +1,4 @@
-FROM golang:1.22-alpine as build
+FROM golang:1.23-alpine as build
 
 ARG TARGETARCH
 ARG RELEASE
@@ -53,7 +53,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
-RUN chmod 1777 /usr/bin
+RUN chmod -R 777 /usr/bin
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio* /usr/bin/

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM golang:1.22-alpine AS build
+FROM golang:1.23-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
@@ -55,7 +55,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
-RUN chmod 1777 /usr/bin
+RUN chmod -R 777 /usr/bin
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio* /usr/bin/

Dockerfile.release.fips

@@ -1,4 +1,4 @@
-FROM golang:1.22-alpine AS build
+FROM golang:1.23-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
@@ -51,7 +51,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
     MINIO_CONFIG_ENV_FILE=config.env
 
-RUN chmod 1777 /usr/bin
+RUN chmod -R 777 /usr/bin
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio* /usr/bin/

Dockerfile.release.old_cpu

@@ -1,4 +1,4 @@
-FROM golang:1.22-alpine AS build
+FROM golang:1.23-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
@@ -53,7 +53,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env \
     MC_CONFIG_DIR=/tmp/.mc
 
-RUN chmod 1777 /usr/bin
+RUN chmod -R 777 /usr/bin
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio* /usr/bin/

Makefile

@@ -24,7 +24,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.2.5
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -60,6 +60,10 @@ test-ilm: install-race
 	@echo "Running ILM tests"
 	@env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh
 
+test-ilm-transition: install-race
+	@echo "Running ILM tiering tests with healing"
+	@env bash $(PWD)/docs/bucket/lifecycle/setup_ilm_transition.sh
+
 test-pbac: install-race
 	@echo "Running bucket policies tests"
 	@env bash $(PWD)/docs/iam/policies/pbac-tests.sh
@@ -101,6 +105,10 @@ test-iam-import-with-missing-entities: install-race ## test import of external i
 	@echo "Test IAM import configurations with missing entities"
 	@env bash $(PWD)/docs/distributed/iam-import-with-missing-entities.sh
 
+test-iam-import-with-openid: install-race
+	@echo "Test IAM import configurations with openid"
+	@env bash $(PWD)/docs/distributed/iam-import-with-openid.sh
+
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
 
@@ -141,6 +149,10 @@ test-multipart: install-race ## test multipart
 	@echo "Test multipart behavior when part files are missing"
 	@(env bash $(PWD)/buildscripts/multipart-quorum-test.sh)
 
+test-timeout: install-race ## test multipart
+	@echo "Test server timeout"
+	@(env bash $(PWD)/buildscripts/test-timeout.sh)
+
 verify: install-race ## verify minio various setups
 	@echo "Verifying build with race"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)

README.md

@@ -4,7 +4,7 @@
 
 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
-MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
+MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. To learn more about what MinIO is doing for AI storage, go to [AI storage documentation](https://min.io/solutions/object-storage-for-ai).
 
 This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
 
@@ -93,7 +93,6 @@ The following table lists supported architectures. Replace the `wget` URL with t
 | 64-bit Intel/AMD               | <https://dl.min.io/server/minio/release/linux-amd64/minio>   |
 | 64-bit ARM                     | <https://dl.min.io/server/minio/release/linux-arm64/minio>   |
 | 64-bit PowerPC LE (ppc64le)    | <https://dl.min.io/server/minio/release/linux-ppc64le/minio> |
-| IBM Z-Series (S390X)           | <https://dl.min.io/server/minio/release/linux-s390x/minio>   |
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 

buildscripts/minio-iam-ldap-upgrade-import-test.sh

@@ -8,7 +8,7 @@
 #
 # This script assumes that LDAP server is at:
 #
-#   `localhost:1389`
+#   `localhost:389`
 #
 # if this is not the case, set the environment variable
 # `_MINIO_LDAP_TEST_SERVER`.
@@ -41,7 +41,7 @@ __init__() {
 	fi
 
 	if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then
-		export _MINIO_LDAP_TEST_SERVER=localhost:1389
+		export _MINIO_LDAP_TEST_SERVER=localhost:389
 		echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER"
 	fi
 
@@ -58,7 +58,7 @@ create_iam_content_in_old_minio() {
 	mc alias set old-minio http://localhost:9000 minioadmin minioadmin
 	mc ready old-minio
 	mc idp ldap add old-minio \
-		server_addr=localhost:1389 \
+		server_addr=localhost:389 \
 		server_insecure=on \
 		lookup_bind_dn=cn=admin,dc=min,dc=io \
 		lookup_bind_password=admin \

buildscripts/test-timeout.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+	set -x
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+trap 'catch $LINENO' ERR
+
+function purge() {
+	rm -rf "$1"
+}
+
+# shellcheck disable=SC2120
+catch() {
+	if [ $# -ne 0 ]; then
+		echo "error on line $1"
+	fi
+
+	echo "Cleaning up instances of MinIO"
+	pkill minio || true
+	pkill -9 minio || true
+	purge "$WORK_DIR"
+	if [ $# -ne 0 ]; then
+		exit $#
+	fi
+}
+
+catch
+
+function gen_put_request() {
+	hdr_sleep=$1
+	body_sleep=$2
+
+	echo "PUT /testbucket/testobject HTTP/1.1"
+	sleep $hdr_sleep
+	echo "Host: foo-header"
+	echo "User-Agent: curl/8.2.1"
+	echo "Accept: */*"
+	echo "Content-Length: 30"
+	echo ""
+
+	sleep $body_sleep
+	echo "random line 0"
+	echo "random line 1"
+	echo ""
+	echo ""
+}
+
+function send_put_object_request() {
+	hdr_timeout=$1
+	body_timeout=$2
+
+	start=$(date +%s)
+	timeout 5m bash -c "gen_put_request $hdr_timeout $body_timeout | netcat 127.0.0.1 $start_port | read" || return -1
+	[ $(($(date +%s) - start)) -gt $((srv_hdr_timeout + srv_idle_timeout + 1)) ] && return -1
+	return 0
+}
+
+function test_minio_with_timeout() {
+	start_port=$1
+
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	export MINIO_CI_CD=1
+
+	mkdir ${WORK_DIR}
+	C_PWD=${PWD}
+	if [ ! -x "$PWD/mc" ]; then
+		MC_BUILD_DIR="mc-$RANDOM"
+		if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+			echo "failed to download https://github.com/minio/mc"
+			purge "${MC_BUILD_DIR}"
+			exit 1
+		fi
+
+		(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+		# remove mc source.
+		purge "${MC_BUILD_DIR}"
+	fi
+
+	"${MINIO[@]}" --address ":$start_port" --read-header-timeout ${srv_hdr_timeout}s --idle-timeout ${srv_idle_timeout}s "${WORK_DIR}/disk/" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 1
+
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	set -e
+
+	"${PWD}/mc" mb minio/testbucket
+	"${PWD}/mc" anonymous set public minio/testbucket
+
+	# slow header writing
+	send_put_object_request 20 0 && exit -1
+	"${PWD}/mc" stat minio/testbucket/testobject && exit -1
+
+	# quick header write and slow bodywrite
+	send_put_object_request 0 40 && exit -1
+	"${PWD}/mc" stat minio/testbucket/testobject && exit -1
+
+	# quick header and body write
+	send_put_object_request 1 1 || exit -1
+	"${PWD}/mc" stat minio/testbucket/testobject || exit -1
+}
+
+function main() {
+	export start_port=$(shuf -i 10000-65000 -n 1)
+	export srv_hdr_timeout=5
+	export srv_idle_timeout=5
+	export -f gen_put_request
+
+	test_minio_with_timeout ${start_port}
+}
+
+main "$@"

cmd/admin-handlers-idp-ldap.go

@@ -190,7 +190,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 //
 // PUT /minio/admin/v3/idp/ldap/add-service-account
 func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
-	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, true)
 	if APIError.Code != "" {
 		writeErrorResponseJSON(ctx, w, APIError, r.URL)
 		return
@@ -499,7 +499,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 
 	dnList := r.Form["userDNs"]
 	isAll := r.Form.Get("all") == "true"
-	onlySelf := !isAll && len(dnList) == 0
+	selfOnly := !isAll && len(dnList) == 0
 
 	if isAll && len(dnList) > 0 {
 		// This should be checked on client side, so return generic error
@@ -527,7 +527,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 			dn = foundResult.NormDN
 		}
 		if dn == cred.ParentUser || dnList[0] == cred.ParentUser {
-			onlySelf = true
+			selfOnly = true
 		}
 	}
 
@@ -538,13 +538,13 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-		DenyOnly:        onlySelf,
+		DenyOnly:        selfOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
 	}
 
-	if onlySelf && len(dnList) == 0 {
+	if selfOnly && len(dnList) == 0 {
 		selfDN := cred.AccessKey
 		if cred.ParentUser != "" {
 			selfDN = cred.ParentUser
@@ -609,10 +609,9 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 				return
 			}
 			for _, sts := range stsKeys {
-				expiryTime := sts.Expiration
 				accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
 					AccessKey:  sts.AccessKey,
-					Expiration: &expiryTime,
+					Expiration: &sts.Expiration,
 				})
 			}
 			// if only STS keys, skip if user has no STS keys
@@ -628,10 +627,9 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 				return
 			}
 			for _, svc := range serviceAccounts {
-				expiryTime := svc.Expiration
 				accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
 					AccessKey:  svc.AccessKey,
-					Expiration: &expiryTime,
+					Expiration: &svc.Expiration,
 				})
 			}
 			// if only service accounts, skip if user has no service accounts

cmd/admin-handlers-users.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
+	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	xldap "github.com/minio/pkg/v3/ldap"
 	"github.com/minio/pkg/v3/policy"
@@ -65,6 +66,17 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// This API only supports removal of internal users not service accounts.
+	ok, _, err = globalIAMSys.IsServiceAccount(accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if ok {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+		return
+	}
+
 	// When the user is root credential you are not allowed to
 	// remove the root user. Also you cannot delete yourself.
 	if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey {
@@ -637,7 +649,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, false)
 	if APIError.Code != "" {
 		writeErrorResponseJSON(ctx, w, APIError, r.URL)
 		return
@@ -1167,6 +1179,172 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
+// ListAccessKeysBulk - GET /minio/admin/v3/list-access-keys-bulk
+func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	users := r.Form["users"]
+	isAll := r.Form.Get("all") == "true"
+	selfOnly := !isAll && len(users) == 0
+
+	if isAll && len(users) > 0 {
+		// This should be checked on client side, so return generic error
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	// Empty user list and not self, list access keys for all users
+	if isAll {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListUsersAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else if len(users) == 1 {
+		if users[0] == cred.AccessKey || users[0] == cred.ParentUser {
+			selfOnly = true
+		}
+	}
+
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.ListServiceAccountsAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        selfOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if selfOnly && len(users) == 0 {
+		selfUser := cred.AccessKey
+		if cred.ParentUser != "" {
+			selfUser = cred.ParentUser
+		}
+		users = append(users, selfUser)
+	}
+
+	var checkedUserList []string
+	if isAll {
+		users, err := globalIAMSys.ListUsers(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		for user := range users {
+			checkedUserList = append(checkedUserList, user)
+		}
+		checkedUserList = append(checkedUserList, globalActiveCred.AccessKey)
+	} else {
+		for _, user := range users {
+			// Validate the user
+			_, ok := globalIAMSys.GetUser(ctx, user)
+			if !ok {
+				continue
+			}
+			checkedUserList = append(checkedUserList, user)
+		}
+	}
+
+	listType := r.Form.Get("listType")
+	var listSTSKeys, listServiceAccounts bool
+	switch listType {
+	case madmin.AccessKeyListUsersOnly:
+		listSTSKeys = false
+		listServiceAccounts = false
+	case madmin.AccessKeyListSTSOnly:
+		listSTSKeys = true
+		listServiceAccounts = false
+	case madmin.AccessKeyListSvcaccOnly:
+		listSTSKeys = false
+		listServiceAccounts = true
+	case madmin.AccessKeyListAll:
+		listSTSKeys = true
+		listServiceAccounts = true
+	default:
+		err := errors.New("invalid list type")
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+
+	accessKeyMap := make(map[string]madmin.ListAccessKeysResp)
+	for _, user := range checkedUserList {
+		accessKeys := madmin.ListAccessKeysResp{}
+		if listSTSKeys {
+			stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, user)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, sts := range stsKeys {
+				accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
+					AccessKey:  sts.AccessKey,
+					Expiration: &sts.Expiration,
+				})
+			}
+			// if only STS keys, skip if user has no STS keys
+			if !listServiceAccounts && len(stsKeys) == 0 {
+				continue
+			}
+		}
+
+		if listServiceAccounts {
+			serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, user)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, svc := range serviceAccounts {
+				accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
+					AccessKey:  svc.AccessKey,
+					Expiration: &svc.Expiration,
+				})
+			}
+			// if only service accounts, skip if user has no service accounts
+			if !listSTSKeys && len(serviceAccounts) == 0 {
+				continue
+			}
+		}
+		accessKeyMap[user] = accessKeys
+	}
+
+	data, err := json.Marshal(accessKeyMap)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}
+
 // AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -1276,7 +1454,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 
 	var buf []byte
 	switch {
-	case accountName == globalActiveCred.AccessKey:
+	case accountName == globalActiveCred.AccessKey || newGlobalAuthZPluginFn() != nil:
+		// For owner account and when plugin authZ is configured always set
+		// effective policy as `consoleAdmin`.
+		//
+		// In the latter case, we let the UI render everything, but individual
+		// actions would fail if not permitted by the external authZ service.
 		for _, policy := range policy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
 				effectivePolicy = policy.Definition
@@ -1397,6 +1580,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
 		return
 	}
+	setReqInfoPolicyName(ctx, name)
 
 	policyDoc, err := globalIAMSys.InfoPolicy(name)
 	if err != nil {
@@ -1500,6 +1684,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	vars := mux.Vars(r)
 	policyName := vars["name"]
+	setReqInfoPolicyName(ctx, policyName)
 
 	if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1532,6 +1717,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 		return
 	}
+	setReqInfoPolicyName(ctx, policyName)
+
+	// Reject policy names with commas.
+	if strings.Contains(policyName, ",") {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidName), r.URL)
+		return
+	}
 
 	// Error out if Content-Length is missing.
 	if r.ContentLength <= 0 {
@@ -1597,6 +1789,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	policyName := vars["policyName"]
 	entityName := vars["userOrGroup"]
 	isGroup := vars["isGroup"] == "true"
+	setReqInfoPolicyName(ctx, policyName)
 
 	if !isGroup {
 		ok, _, err := globalIAMSys.IsTempUser(entityName)
@@ -1682,7 +1875,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	}))
 }
 
-// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/polciy-entities?policy=xxx&user=xxx&group=xxx
+// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/policy-entities?policy=xxx&user=xxx&group=xxx
 func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -1784,6 +1977,7 @@ func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+	setReqInfoPolicyName(ctx, strings.Join(addedOrRemoved, ","))
 
 	respBody := madmin.PolicyAssociationResp{
 		UpdatedAt: updatedAt,
@@ -1838,6 +2032,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
 	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 	// Initialize a zip writer which will provide a zipped content
@@ -2060,17 +2255,13 @@ func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) {
 	ctx := r.Context()
 
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
+	// Validate signature, permissions and get current object layer instance.
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportIAMAction)
 	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
-	if s3Err != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
-		return
-	}
+
 	data, err := io.ReadAll(r.Body)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
@@ -2160,38 +2351,12 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
 					return
 				}
 
-				if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
-					// Incoming access key matches parent user then we should
-					// reject password change requests.
-					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
-					return
-				}
-
 				// Check if accessKey has beginning and end space characters, this only applies to new users.
 				if !exists && hasSpaceBE(accessKey) {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allUsersFile, accessKey), r.URL)
 					return
 				}
 
-				checkDenyOnly := false
-				if accessKey == cred.AccessKey {
-					// Check that there is no explicit deny - otherwise it's allowed
-					// to change one's own password.
-					checkDenyOnly = true
-				}
-
-				if !globalIAMSys.IsAllowed(policy.Args{
-					AccountName:     cred.AccessKey,
-					Groups:          cred.Groups,
-					Action:          policy.CreateUserAdminAction,
-					ConditionValues: getConditionValues(r, "", cred),
-					IsOwner:         owner,
-					Claims:          cred.Claims,
-					DenyOnly:        checkDenyOnly,
-				}) {
-					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allUsersFile, accessKey), r.URL)
-					return
-				}
 				if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
 					failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err})
 				} else {
@@ -2291,17 +2456,6 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
 					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 					return
 				}
-				if !globalIAMSys.IsAllowed(policy.Args{
-					AccountName:     cred.AccessKey,
-					Groups:          cred.Groups,
-					Action:          policy.CreateServiceAccountAdminAction,
-					ConditionValues: getConditionValues(r, "", cred),
-					IsOwner:         owner,
-					Claims:          cred.Claims,
-				}) {
-					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allSvcAcctsFile, user), r.URL)
-					return
-				}
 				updateReq := true
 				_, _, err = globalIAMSys.GetServiceAccount(ctx, svcAcctReq.AccessKey)
 				if err != nil {
@@ -2529,7 +2683,7 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
 	return nil
 }
 
-func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
+func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
 	ctx := r.Context()
 
 	// Get current object layer instance.
@@ -2596,6 +2750,14 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
 	}
 
+	denyOnly := (targetUser == cred.AccessKey || targetUser == cred.ParentUser)
+	if ldap && !denyOnly {
+		res, _ := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser)
+		if res.NormDN == cred.ParentUser {
+			denyOnly = true
+		}
+	}
+
 	// Check if action is allowed if creating access key for another user
 	// Check if action is explicitly denied if for self
 	if !globalIAMSys.IsAllowed(policy.Args{
@@ -2605,7 +2767,7 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 		ConditionValues: condValues,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-		DenyOnly:        (targetUser == cred.AccessKey || targetUser == cred.ParentUser),
+		DenyOnly:        denyOnly,
 	}) {
 		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
 	}
@@ -2622,3 +2784,10 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 
 	return ctx, cred, opts, createReq, targetUser, APIError{}
 }
+
+// setReqInfoPolicyName will set the given policyName as a tag on the context's request info,
+// so that it appears in audit logs.
+func setReqInfoPolicyName(ctx context.Context, policyName string) {
+	reqInfo := logger.GetReqInfo(ctx)
+	reqInfo.SetTags("policyName", policyName)
+}

cmd/admin-handlers-users_test.go

@@ -338,16 +338,24 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -443,7 +451,7 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
     "s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::testbucket/*"
+    "arn:aws:s3:::testbucket"
    ]
   }
  ]
@@ -560,16 +568,24 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -670,16 +686,24 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 
 	// Check that default policies can be overwritten.
 	err = s.adm.AddCannedPolicy(ctx, "readwrite", policyBytes)
@@ -692,6 +716,12 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
 		c.Fatalf("policy info err: %v", err)
 	}
 
+	// Check that policy with comma is rejected.
+	err = s.adm.AddCannedPolicy(ctx, "invalid,policy", policyBytes)
+	if err == nil {
+		c.Fatalf("invalid policy created successfully")
+	}
+
 	infoStr := string(info)
 	if !strings.Contains(infoStr, `"s3:PutObject"`) || !strings.Contains(infoStr, ":"+bucket+"/") {
 		c.Fatalf("policy contains unexpected content!")
@@ -715,16 +745,24 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -879,16 +917,24 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -963,16 +1009,24 @@ func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -1045,16 +1099,24 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
     "s3:ListBucket"
    ],
+   "Resource": [
+    "arn:aws:s3:::%s"
+   ]
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
    "Resource": [
     "arn:aws:s3:::%s/*"
    ]
   }
  ]
-}`, bucket))
+}`, bucket, bucket))
 	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
@@ -1602,7 +1664,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
     "s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::%s/*"
+    "arn:aws:s3:::%s"
    ]
   }
  ]

cmd/admin-handlers.go

@@ -93,11 +93,18 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
+	if globalInplaceUpdateDisabled {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
 	}
 
+	if currentReleaseTime.IsZero() || currentReleaseTime.Equal(timeSentinel) {
+		apiErr := errorCodes.ToAPIErr(ErrMethodNotAllowed)
+		apiErr.Description = fmt.Sprintf("unable to perform in-place update, release time is unrecognized: %s", currentReleaseTime)
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
+		return
+	}
+
 	vars := mux.Vars(r)
 	updateURL := vars["updateURL"]
 	dryRun := r.Form.Get("dry-run") == "true"
@@ -110,6 +117,11 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		}
 	}
 
+	local := globalLocalNodeName
+	if local == "" {
+		local = "127.0.0.1"
+	}
+
 	u, err := url.Parse(updateURL)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -128,6 +140,39 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		return
 	}
 
+	updateStatus := madmin.ServerUpdateStatusV2{
+		DryRun:  dryRun,
+		Results: make([]madmin.ServerPeerUpdateStatus, 0, len(globalNotificationSys.allPeerClients)),
+	}
+	peerResults := make(map[string]madmin.ServerPeerUpdateStatus, len(globalNotificationSys.allPeerClients))
+	failedClients := make(map[int]bool, len(globalNotificationSys.allPeerClients))
+
+	if lrTime.Sub(currentReleaseTime) <= 0 {
+		updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
+			Host:           local,
+			Err:            fmt.Sprintf("server is running the latest version: %s", Version),
+			CurrentVersion: Version,
+		})
+
+		for _, client := range globalNotificationSys.peerClients {
+			updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
+				Host:           client.String(),
+				Err:            fmt.Sprintf("server is running the latest version: %s", Version),
+				CurrentVersion: Version,
+			})
+		}
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(updateStatus)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		writeSuccessResponseJSON(w, jsonBytes)
+		return
+	}
+
 	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
 	// Download Binary Once
 	binC, bin, err := downloadBinary(u, mode)
@@ -137,16 +182,6 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	updateStatus := madmin.ServerUpdateStatusV2{DryRun: dryRun}
-	peerResults := make(map[string]madmin.ServerPeerUpdateStatus)
-
-	local := globalLocalNodeName
-	if local == "" {
-		local = "127.0.0.1"
-	}
-
-	failedClients := make(map[int]struct{})
-
 	if globalIsDistErasure {
 		// Push binary to other servers
 		for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
@@ -156,7 +191,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 					Err:            nerr.Err.Error(),
 					CurrentVersion: Version,
 				}
-				failedClients[idx] = struct{}{}
+				failedClients[idx] = true
 			} else {
 				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
 					Host:           nerr.Host.String(),
@@ -167,25 +202,17 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		}
 	}
 
-	if lrTime.Sub(currentReleaseTime) > 0 {
-		if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
-			peerResults[local] = madmin.ServerPeerUpdateStatus{
-				Host:           local,
-				Err:            err.Error(),
-				CurrentVersion: Version,
-			}
-		} else {
-			peerResults[local] = madmin.ServerPeerUpdateStatus{
-				Host:           local,
-				CurrentVersion: Version,
-				UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
-			}
+	if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
+		peerResults[local] = madmin.ServerPeerUpdateStatus{
+			Host:           local,
+			Err:            err.Error(),
+			CurrentVersion: Version,
 		}
 	} else {
 		peerResults[local] = madmin.ServerPeerUpdateStatus{
 			Host:           local,
-			Err:            fmt.Sprintf("server is already running the latest version: %s", Version),
 			CurrentVersion: Version,
+			UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 		}
 	}
 
@@ -193,8 +220,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		if globalIsDistErasure {
 			ng := WithNPeers(len(globalNotificationSys.peerClients))
 			for idx, client := range globalNotificationSys.peerClients {
-				_, ok := failedClients[idx]
-				if ok {
+				if failedClients[idx] {
 					continue
 				}
 				client := client
@@ -240,14 +266,14 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 		startTime := time.Now().Add(restartUpdateDelay)
 		ng := WithNPeers(len(globalNotificationSys.peerClients))
 		for idx, client := range globalNotificationSys.peerClients {
-			_, ok := failedClients[idx]
-			if ok {
+			if failedClients[idx] {
 				continue
 			}
 			client := client
 			ng.Go(ctx, func() error {
 				prs, ok := peerResults[client.String()]
-				if ok && prs.CurrentVersion != prs.UpdatedVersion && prs.UpdatedVersion != "" {
+				// We restart only on success, not for any failures.
+				if ok && prs.Err == "" {
 					return client.SignalService(serviceRestart, "", dryRun, &startTime)
 				}
 				return nil
@@ -284,7 +310,9 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 	writeSuccessResponseJSON(w, jsonBytes)
 
 	if !dryRun {
-		if lrTime.Sub(currentReleaseTime) > 0 {
+		prs, ok := peerResults[local]
+		// We restart only on success, not for any failures.
+		if ok && prs.Err == "" {
 			globalServiceSignalCh <- serviceRestart
 		}
 	}

cmd/admin-heal-ops.go

@@ -806,18 +806,20 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
 }
 
-func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+func (h *healSequence) healItems(objAPI ObjectLayer) error {
 	if h.clientToken == bgHealingUUID {
 		// For background heal do nothing.
 		return nil
 	}
 
-	if err := h.healDiskMeta(objAPI); err != nil {
-		return err
+	if h.bucket == "" { // heal internal meta only during a site-wide heal
+		if err := h.healDiskMeta(objAPI); err != nil {
+			return err
+		}
 	}
 
 	// Heal buckets and objects
-	return h.healBuckets(objAPI, bucketsOnly)
+	return h.healBuckets(objAPI)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -828,8 +830,7 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
-	bucketsOnly := false // Heals buckets and objects also.
-	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
+	h.traverseAndHealDoneCh <- h.healItems(objAPI)
 	xioutil.SafeClose(h.traverseAndHealDoneCh)
 }
 
@@ -840,6 +841,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
+		h.settings.Recursive = true
 		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
@@ -856,14 +858,14 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
+func (h *healSequence) healBuckets(objAPI ObjectLayer) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(objAPI, h.bucket, bucketsOnly)
+		return h.healBucket(objAPI, h.bucket, false)
 	}
 
 	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
@@ -877,7 +879,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	})
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
+		if err = h.healBucket(objAPI, bucket.Name, false); err != nil {
 			return err
 		}
 	}
@@ -895,16 +897,6 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 		return nil
 	}
 
-	if !h.settings.Recursive {
-		if h.object != "" {
-			if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
 	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}

cmd/admin-router.go

@@ -244,6 +244,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// STS accounts ops
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}")
 
+		// Access key (service account/STS) operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-access-keys-bulk").HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysBulk)).Queries("listType", "{listType:.*}")
+
 		// Info policy IAM latest
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 		// List policies latest

cmd/api-errors.go

@@ -213,6 +213,7 @@ const (
 	ErrPolicyAlreadyAttached
 	ErrPolicyNotAttached
 	ErrExcessData
+	ErrPolicyInvalidName
 	// Add new error codes here.
 
 	// SSE-S3/SSE-KMS related API errors
@@ -561,6 +562,11 @@ var errorCodes = errorCodeMap{
 		Description:    "More data provided than indicated content length",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrPolicyInvalidName: {
+		Code:           "PolicyInvalidName",
+		Description:    "Policy name may not contain comma",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrPolicyTooLarge: {
 		Code:           "PolicyTooLarge",
 		Description:    "Policy exceeds the maximum allowed document size.",
@@ -978,7 +984,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationNoExistingObjects: {
 		Code:           "XMinioReplicationNoExistingObjects",
-		Description:    "No matching ExistingsObjects rule enabled",
+		Description:    "No matching ExistingObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrRemoteTargetDenyAddError: {

cmd/api-response.go

@@ -593,8 +593,6 @@ func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, v
 			for k, v := range cleanReservedKeys(object.UserDefined) {
 				content.UserMetadata.Set(k, v)
 			}
-
-			content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
 			content.Internal = &ObjectInternalInfo{
 				K: object.DataBlocks,
 				M: object.ParityBlocks,
@@ -729,7 +727,6 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
 				for k, v := range cleanReservedKeys(object.UserDefined) {
 					content.UserMetadata.Set(k, v)
 				}
-				content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
 				content.Internal = &ObjectInternalInfo{
 					K: object.DataBlocks,
 					M: object.ParityBlocks,

cmd/auth-handler.go

@@ -222,7 +222,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }
 
-func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
+func getClaimsFromTokenWithSecret(token, secret string) (*xjwt.MapClaims, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -244,7 +244,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 
 	// If AuthZPlugin is set, return without any further checks.
 	if newGlobalAuthZPluginFn() != nil {
-		return claims.Map(), nil
+		return claims, nil
 	}
 
 	// Check if a session policy is set. If so, decode it here.
@@ -263,12 +263,16 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
 	}
 
-	return claims.Map(), nil
+	return claims, nil
 }
 
 // Fetch claims in the security token returned by the client.
 func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+	jwtClaims, err := getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+	return jwtClaims.Map(), nil
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
@@ -319,7 +323,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		if err != nil {
 			return nil, toAPIErrorCode(r.Context(), err)
 		}
-		return claims, ErrNone
+		return claims.Map(), ErrNone
 	}
 
 	claims := xjwt.NewMapClaims()

cmd/background-newdisks-heal-ops.go

@@ -223,9 +223,6 @@ func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) {
 // update will update the tracker on the disk.
 // If the tracker has been deleted an error is returned.
 func (h *healingTracker) update(ctx context.Context) error {
-	if h.disk.Healing() == nil {
-		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
-	}
 	h.mu.Lock()
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()

cmd/batch-expire.go

@@ -27,7 +27,6 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -389,9 +388,12 @@ func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
 
 func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
 	vc, _ := globalBucketVersioningSys.Get(r.Bucket)
-	retryAttempts := r.Retry.Attempts
+	retryAttempts := job.Expire.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchExpireJobDefaultRetries
+	}
 	delay := job.Expire.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchExpireJobDefaultRetryDelay
 	}
 
@@ -557,9 +559,13 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	results := make(chan itemOrErr[ObjectInfo], workerSize)
 	go func() {
-		for _, prefix := range r.Prefix.F() {
+		prefixes := r.Prefix.F()
+		if len(prefixes) == 0 {
+			prefixes = []string{""}
+		}
+		for _, prefix := range prefixes {
 			prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
-			err := api.Walk(ctx, r.Bucket, strings.TrimSpace(prefix), prefixResultCh, WalkOptions{
+			err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
 				Marker:       lastObject,
 				LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
 				VersionsSort: WalkVersionsSortDesc,

cmd/batch-handlers.go

@@ -276,8 +276,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	globalBatchJobsMetrics.save(job.ID, ri)
 
+	retryAttempts := job.Replicate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchReplJobDefaultRetries
+	}
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchReplJobDefaultRetryDelay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
@@ -297,12 +301,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return true
 		}
 
-		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(oi.ModTime) {
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(oi.ModTime) {
 			// skip all objects that are created before the specified time.
 			return true
 		}
 
-		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(oi.ModTime) {
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(oi.ModTime) {
 			// skip all objects that are created after the specified time.
 			return true
 		}
@@ -377,7 +381,6 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		return err
 	}
 
-	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
@@ -394,7 +397,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			}
 			for _, prefix := range prefixes {
 				prefixObjInfoCh := c.ListObjects(ctx, r.Source.Bucket, miniogo.ListObjectsOptions{
-					Prefix:       strings.TrimSpace(prefix),
+					Prefix:       prefix,
 					WithVersions: minioSrc,
 					Recursive:    true,
 					WithMetadata: true,
@@ -772,22 +775,10 @@ func (ri *batchJobInfo) loadOrInit(ctx context.Context, api ObjectLayer, job Bat
 		switch {
 		case job.Replicate != nil:
 			ri.Version = batchReplVersionV1
-			ri.RetryAttempts = batchReplJobDefaultRetries
-			if job.Replicate.Flags.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
-			}
 		case job.KeyRotate != nil:
 			ri.Version = batchKeyRotateVersionV1
-			ri.RetryAttempts = batchKeyRotateJobDefaultRetries
-			if job.KeyRotate.Flags.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
-			}
 		case job.Expire != nil:
 			ri.Version = batchExpireVersionV1
-			ri.RetryAttempts = batchExpireJobDefaultRetries
-			if job.Expire.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.Expire.Retry.Attempts
-			}
 		}
 		return nil
 	}
@@ -1034,10 +1025,15 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
+	retryAttempts := job.Replicate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchReplJobDefaultRetries
+	}
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay < time.Second {
+	if delay <= 0 {
 		delay = batchReplJobDefaultRetryDelay
 	}
+
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	selectObj := func(info FileInfo) (ok bool) {
@@ -1051,12 +1047,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			return false
 		}
 
-		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(info.ModTime) {
 			// skip all objects that are created before the specified time.
 			return false
 		}
 
-		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(info.ModTime) {
 			// skip all objects that are created after the specified time.
 			return false
 		}
@@ -1125,7 +1121,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
@@ -1214,7 +1209,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			}
 			for _, prefix := range prefixes {
 				prefixWalkCh := make(chan itemOrErr[ObjectInfo], 100)
-				if err := api.Walk(ctx, r.Source.Bucket, strings.TrimSpace(prefix), prefixWalkCh, WalkOptions{
+				if err := api.Walk(ctx, r.Source.Bucket, prefix, prefixWalkCh, WalkOptions{
 					Marker:   lastObject,
 					Filter:   selectObj,
 					AskDisks: walkQuorum,
@@ -1582,9 +1577,6 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 	}
 
 	jobType := r.Form.Get("jobType")
-	if jobType == "" {
-		jobType = string(madmin.BatchJobReplicate)
-	}
 
 	resultCh := make(chan itemOrErr[ObjectInfo])
 
@@ -1613,7 +1605,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 			continue
 		}
 
-		if jobType == string(req.Type()) {
+		if jobType == string(req.Type()) || jobType == "" {
 			listResult.Jobs = append(listResult.Jobs, madmin.BatchJobResult{
 				ID:      req.ID,
 				Type:    req.Type(),
@@ -1994,7 +1986,6 @@ func (j *BatchJobPool) AddWorker() {
 					}
 				}
 			}
-			job.delete(j.ctx, j.objLayer)
 			j.canceler(job.ID, false)
 		case <-j.workerKillCh:
 			return
@@ -2055,7 +2046,9 @@ func (j *BatchJobPool) canceler(jobID string, cancel bool) error {
 			canceler()
 		}
 	}
-	delete(j.jobCancelers, jobID)
+	if cancel {
+		delete(j.jobCancelers, jobID)
+	}
 	return nil
 }
 
@@ -2181,11 +2174,42 @@ func (m *batchJobMetrics) purgeJobMetrics() {
 			m.RUnlock()
 			for _, jobID := range toDeleteJobMetrics {
 				m.delete(jobID)
+				j := BatchJobRequest{
+					ID: jobID,
+				}
+				j.delete(GlobalContext, newObjectLayerFn())
 			}
 		}
 	}
 }
 
+// load metrics from disk on startup
+func (m *batchJobMetrics) init(ctx context.Context, objectAPI ObjectLayer) error {
+	resultCh := make(chan itemOrErr[ObjectInfo])
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, resultCh, WalkOptions{}); err != nil {
+		return err
+	}
+
+	for result := range resultCh {
+		if result.Err != nil {
+			return result.Err
+		}
+		ri := &batchJobInfo{}
+		if err := ri.loadByPath(ctx, objectAPI, result.Item.Name); err != nil {
+			if !errors.Is(err, errNoSuchJob) {
+				batchLogIf(ctx, err)
+			}
+			continue
+		}
+		m.metrics[ri.JobID] = ri
+	}
+	return nil
+}
+
 func (m *batchJobMetrics) delete(jobID string) {
 	m.Lock()
 	defer m.Unlock()

cmd/batch-rotate.go

@@ -267,8 +267,12 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
+	retryAttempts := job.KeyRotate.Flags.Retry.Attempts
+	if retryAttempts <= 0 {
+		retryAttempts = batchKeyRotateJobDefaultRetries
+	}
 	delay := job.KeyRotate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay <= 0 {
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
 
@@ -354,7 +358,6 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		return err
 	}
 
-	retryAttempts := ri.RetryAttempts
 	ctx, cancel := context.WithCancel(ctx)
 
 	results := make(chan itemOrErr[ObjectInfo], 100)

cmd/bootstrap-peer-server.go

@@ -183,11 +183,19 @@ var binaryChecksum = getBinaryChecksum()
 
 func getBinaryChecksum() string {
 	mw := md5.New()
-	b, err := os.Open(os.Args[0])
-	if err == nil {
-		defer b.Close()
-		io.Copy(mw, b)
+	binPath, err := os.Executable()
+	if err != nil {
+		logger.Error("Calculating checksum failed: %s", err)
+		return "00000000000000000000000000000000"
 	}
+	b, err := os.Open(binPath)
+	if err != nil {
+		logger.Error("Calculating checksum failed: %s", err)
+		return "00000000000000000000000000000000"
+	}
+
+	defer b.Close()
+	io.Copy(mw, b)
 	return hex.EncodeToString(mw.Sum(nil))
 }
 

cmd/bucket-handlers.go

@@ -1338,6 +1338,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					return
 				}
 			}
+			opts.EncryptFn = metadataEncrypter(objectEncryptionKey)
 			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1809,6 +1810,10 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
+	// Audit log tags.
+	reqInfo := logger.GetReqInfo(ctx)
+	reqInfo.SetTags("retention", config.String())
+
 	configData, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/bucket-lifecycle.go

@@ -71,7 +71,7 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
-func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string) madmin.TraceInfo {
+func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string, err string) madmin.TraceInfo {
 	sz, _ := oi.GetActualSize()
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceILM,
@@ -81,18 +81,22 @@ func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event
 		Duration:  duration,
 		Path:      pathJoin(oi.Bucket, oi.Name),
 		Bytes:     sz,
-		Error:     "",
+		Error:     err,
 		Message:   getSource(4),
 		Custom:    metadata,
 	}
 }
 
-func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string) {
+func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string, err error) {
 	startTime := time.Now()
-	return func(event string, metadata map[string]string) {
+	return func(event string, metadata map[string]string, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
-			globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata))
+			e := ""
+			if err != nil {
+				e = err.Error()
+			}
+			globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata, e))
 		}
 	}
 }
@@ -348,7 +352,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 				traceFn := globalLifecycleSys.trace(oi)
 				if !oi.TransitionedObject.FreeVersion {
 					// nothing to be done
-					return
+					continue
 				}
 
 				ignoreNotFoundErr := func(err error) error {
@@ -362,7 +366,8 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
 				if ignoreNotFoundErr(err) != nil {
 					transitionLogIf(es.ctx, err)
-					return
+					traceFn(ILMFreeVersionDelete, nil, err)
+					continue
 				}
 
 				// Remove this free version

cmd/bucket-listobjects-handlers.go

@@ -245,7 +245,7 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
 
 func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) {
 	subToken, nodeIndex := parseRequestToken(token)
-	if nodeIndex > 0 {
+	if nodeIndex >= 0 {
 		return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex)
 	}
 	return subToken, false

cmd/bucket-replication.go

@@ -53,8 +53,6 @@ import (
 	"github.com/minio/minio/internal/once"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 )
 
 const (
@@ -780,8 +778,9 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
 
 	for k, v := range objInfo.UserDefined {
+		_, isValidSSEHeader := validSSEReplicationHeaders[k]
 		// In case of SSE-C objects copy the allowed internal headers as well
-		if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
+		if !isSSEC || !isValidSSEHeader {
 			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 				continue
 			}
@@ -789,7 +788,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 				continue
 			}
 		}
-		if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
+		if isValidSSEHeader {
 			meta[validSSEReplicationHeaders[k]] = v
 		} else {
 			meta[k] = v
@@ -889,6 +888,14 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
+
+	if crypto.S3KMS.IsEncrypted(objInfo.UserDefined) {
+		sseEnc, err := encrypt.NewSSEKMS(objInfo.KMSKeyID(), nil)
+		if err != nil {
+			return putOpts, err
+		}
+		putOpts.ServerSideEncryption = sseEnc
+	}
 	return
 }
 

cmd/bucket-stats.go

@@ -108,18 +108,18 @@ func (l ReplicationLastHour) merge(o ReplicationLastHour) (merged ReplicationLas
 
 // Add  a new duration data
 func (l *ReplicationLastHour) addsize(sz int64) {
-	min := time.Now().Unix() / 60
-	l.forwardTo(min)
-	winIdx := min % 60
-	l.Totals[winIdx].merge(AccElem{Total: min, Size: sz, N: 1})
-	l.LastMin = min
+	minutes := time.Now().Unix() / 60
+	l.forwardTo(minutes)
+	winIdx := minutes % 60
+	l.Totals[winIdx].merge(AccElem{Total: minutes, Size: sz, N: 1})
+	l.LastMin = minutes
 }
 
 // Merge all recorded counts of last hour into one
 func (l *ReplicationLastHour) getTotal() AccElem {
 	var res AccElem
-	min := time.Now().Unix() / 60
-	l.forwardTo(min)
+	minutes := time.Now().Unix() / 60
+	l.forwardTo(minutes)
 	for _, elem := range l.Totals[:] {
 		res.merge(elem)
 	}

cmd/common-main.go

@@ -91,9 +91,6 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
-	go globalBatchJobsMetrics.purgeJobMetrics()
-
 	t, _ := minioVersionToReleaseTime(Version)
 	if !t.IsZero() {
 		globalVersionUnix = uint64(t.Unix())
@@ -138,6 +135,9 @@ func minioConfigToConsoleFeatures() {
 	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
 		os.Setenv("CONSOLE_BROWSER_REDIRECT_URL", value)
 	}
+	if value := env.Get(config.EnvConsoleDebugLogLevel, ""); value != "" {
+		os.Setenv("CONSOLE_DEBUG_LOGLEVEL", value)
+	}
 	// pass the console subpath configuration
 	if globalBrowserRedirectURL != nil {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))

cmd/data-scanner.go

@@ -1325,10 +1325,13 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			traceFn(ILMExpiry, nil, nil)
 			return false
 		}
 		// Assume it is still there.
-		ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
+		err := fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err)
+		ilmLogOnceIf(ctx, err, "non-transition-expiry"+obj.Name)
+		traceFn(ILMExpiry, nil, err)
 		return false
 	}
 	if dobj.Name == "" {
@@ -1549,7 +1552,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string)) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1567,5 +1570,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
 		VersionID: oi.VersionID,
 		Tags:      tags,
 	})
-	traceFn(event, tags)
+	traceFn(event, tags, nil)
 }

cmd/data-usage-cache.go

@@ -39,6 +39,8 @@ import (
 	"github.com/valyala/bytebufferpool"
 )
 
+//msgp:clearomitted
+
 //go:generate msgp -file $GOFILE -unexported
 
 // dataUsageHash is the hash type used.
@@ -57,15 +59,14 @@ type versionsHistogram [dataUsageVersionLen]uint64
 type dataUsageEntry struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
-	Size             int64                `msg:"sz"`
-	Objects          uint64               `msg:"os"`
-	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
-	DeleteMarkers    uint64               `msg:"dms"`
-	ObjSizes         sizeHistogram        `msg:"szs"`
-	ObjVersions      versionsHistogram    `msg:"vh"`
-	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
-	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
-	Compacted        bool                 `msg:"c"`
+	Size          int64             `msg:"sz"`
+	Objects       uint64            `msg:"os"`
+	Versions      uint64            `msg:"vs"` // Versions that are not delete markers.
+	DeleteMarkers uint64            `msg:"dms"`
+	ObjSizes      sizeHistogram     `msg:"szs"`
+	ObjVersions   versionsHistogram `msg:"vh"`
+	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
+	Compacted     bool              `msg:"c"`
 }
 
 // allTierStats is a collection of per-tier stats across all configured remote
@@ -135,93 +136,6 @@ func (ts tierStats) add(u tierStats) tierStats {
 	}
 }
 
-//msgp:tuple replicationStatsV1
-type replicationStatsV1 struct {
-	PendingSize          uint64
-	ReplicatedSize       uint64
-	FailedSize           uint64
-	ReplicaSize          uint64
-	FailedCount          uint64
-	PendingCount         uint64
-	MissedThresholdSize  uint64
-	AfterThresholdSize   uint64
-	MissedThresholdCount uint64
-	AfterThresholdCount  uint64
-}
-
-func (rsv1 replicationStatsV1) Empty() bool {
-	return rsv1.ReplicatedSize == 0 &&
-		rsv1.FailedSize == 0 &&
-		rsv1.FailedCount == 0
-}
-
-//msgp:tuple replicationStats
-type replicationStats struct {
-	PendingSize          uint64
-	ReplicatedSize       uint64
-	FailedSize           uint64
-	FailedCount          uint64
-	PendingCount         uint64
-	MissedThresholdSize  uint64
-	AfterThresholdSize   uint64
-	MissedThresholdCount uint64
-	AfterThresholdCount  uint64
-	ReplicatedCount      uint64
-}
-
-func (rs replicationStats) Empty() bool {
-	return rs.ReplicatedSize == 0 &&
-		rs.FailedSize == 0 &&
-		rs.FailedCount == 0
-}
-
-type replicationAllStats struct {
-	Targets      map[string]replicationStats `msg:"t,omitempty"`
-	ReplicaSize  uint64                      `msg:"r,omitempty"`
-	ReplicaCount uint64                      `msg:"rc,omitempty"`
-}
-
-//msgp:tuple replicationAllStatsV1
-type replicationAllStatsV1 struct {
-	Targets      map[string]replicationStats
-	ReplicaSize  uint64 `msg:"ReplicaSize,omitempty"`
-	ReplicaCount uint64 `msg:"ReplicaCount,omitempty"`
-}
-
-// empty returns true if the replicationAllStats is empty (contains no entries).
-func (r *replicationAllStats) empty() bool {
-	if r == nil {
-		return true
-	}
-	if r.ReplicaSize != 0 || r.ReplicaCount != 0 {
-		return false
-	}
-	for _, v := range r.Targets {
-		if !v.Empty() {
-			return false
-		}
-	}
-	return true
-}
-
-// clone creates a deep-copy clone.
-func (r *replicationAllStats) clone() *replicationAllStats {
-	if r == nil {
-		return nil
-	}
-
-	// Shallow copy
-	dst := *r
-
-	// Copy individual targets.
-	dst.Targets = make(map[string]replicationStats, len(r.Targets))
-	for k, v := range r.Targets {
-		dst.Targets[k] = v
-	}
-
-	return &dst
-}
-
 //msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 //msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
@@ -237,62 +151,54 @@ type dataUsageEntryV2 struct {
 //msgp:tuple dataUsageEntryV3
 type dataUsageEntryV3 struct {
 	// These fields do no include any children.
-	Size                   int64
-	ReplicatedSize         uint64
-	ReplicationPendingSize uint64
-	ReplicationFailedSize  uint64
-	ReplicaSize            uint64
-	Objects                uint64
-	ObjSizes               sizeHistogram
-	Children               dataUsageHashMap
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
+	Children dataUsageHashMap
 }
 
 //msgp:tuple dataUsageEntryV4
 type dataUsageEntryV4 struct {
 	Children dataUsageHashMap
 	// These fields do no include any children.
-	Size             int64
-	Objects          uint64
-	ObjSizes         sizeHistogram
-	ReplicationStats replicationStatsV1
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
 }
 
 //msgp:tuple dataUsageEntryV5
 type dataUsageEntryV5 struct {
 	Children dataUsageHashMap
 	// These fields do no include any children.
-	Size             int64
-	Objects          uint64
-	Versions         uint64 // Versions that are not delete markers.
-	ObjSizes         sizeHistogram
-	ReplicationStats *replicationStatsV1
-	Compacted        bool
+	Size      int64
+	Objects   uint64
+	Versions  uint64 // Versions that are not delete markers.
+	ObjSizes  sizeHistogram
+	Compacted bool
 }
 
 //msgp:tuple dataUsageEntryV6
 type dataUsageEntryV6 struct {
 	Children dataUsageHashMap
 	// These fields do no include any children.
-	Size             int64
-	Objects          uint64
-	Versions         uint64 // Versions that are not delete markers.
-	ObjSizes         sizeHistogram
-	ReplicationStats *replicationAllStatsV1
-	Compacted        bool
+	Size      int64
+	Objects   uint64
+	Versions  uint64 // Versions that are not delete markers.
+	ObjSizes  sizeHistogram
+	Compacted bool
 }
 
 type dataUsageEntryV7 struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
-	Size             int64                `msg:"sz"`
-	Objects          uint64               `msg:"os"`
-	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
-	DeleteMarkers    uint64               `msg:"dms"`
-	ObjSizes         sizeHistogramV1      `msg:"szs"`
-	ObjVersions      versionsHistogram    `msg:"vh"`
-	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
-	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
-	Compacted        bool                 `msg:"c"`
+	Size          int64             `msg:"sz"`
+	Objects       uint64            `msg:"os"`
+	Versions      uint64            `msg:"vs"` // Versions that are not delete markers.
+	DeleteMarkers uint64            `msg:"dms"`
+	ObjSizes      sizeHistogramV1   `msg:"szs"`
+	ObjVersions   versionsHistogram `msg:"vh"`
+	AllTierStats  *allTierStats     `msg:"ats,omitempty"`
+	Compacted     bool              `msg:"c"`
 }
 
 // dataUsageCache contains a cache of data usage entries latest version.
@@ -373,29 +279,6 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.ObjSizes.add(summary.totalSize)
 	e.ObjVersions.add(summary.versions)
 
-	if e.ReplicationStats == nil {
-		e.ReplicationStats = &replicationAllStats{
-			Targets: make(map[string]replicationStats),
-		}
-	} else if e.ReplicationStats.Targets == nil {
-		e.ReplicationStats.Targets = make(map[string]replicationStats)
-	}
-	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
-	e.ReplicationStats.ReplicaCount += uint64(summary.replicaCount)
-
-	for arn, st := range summary.replTargetStats {
-		tgtStat, ok := e.ReplicationStats.Targets[arn]
-		if !ok {
-			tgtStat = replicationStats{}
-		}
-		tgtStat.PendingSize += uint64(st.pendingSize)
-		tgtStat.FailedSize += uint64(st.failedSize)
-		tgtStat.ReplicatedSize += uint64(st.replicatedSize)
-		tgtStat.ReplicatedCount += uint64(st.replicatedCount)
-		tgtStat.FailedCount += st.failedCount
-		tgtStat.PendingCount += st.pendingCount
-		e.ReplicationStats.Targets[arn] = tgtStat
-	}
 	if len(summary.tiers) != 0 {
 		if e.AllTierStats == nil {
 			e.AllTierStats = newAllTierStats()
@@ -410,26 +293,6 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Versions += other.Versions
 	e.DeleteMarkers += other.DeleteMarkers
 	e.Size += other.Size
-	if other.ReplicationStats != nil {
-		if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
-		} else if e.ReplicationStats.Targets == nil {
-			e.ReplicationStats.Targets = make(map[string]replicationStats)
-		}
-		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
-		e.ReplicationStats.ReplicaCount += other.ReplicationStats.ReplicaCount
-		for arn, stat := range other.ReplicationStats.Targets {
-			st := e.ReplicationStats.Targets[arn]
-			e.ReplicationStats.Targets[arn] = replicationStats{
-				PendingSize:     stat.PendingSize + st.PendingSize,
-				FailedSize:      stat.FailedSize + st.FailedSize,
-				ReplicatedSize:  stat.ReplicatedSize + st.ReplicatedSize,
-				PendingCount:    stat.PendingCount + st.PendingCount,
-				FailedCount:     stat.FailedCount + st.FailedCount,
-				ReplicatedCount: stat.ReplicatedCount + st.ReplicatedCount,
-			}
-		}
-	}
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
@@ -490,10 +353,7 @@ func (e dataUsageEntry) clone() dataUsageEntry {
 		}
 		e.Children = ch
 	}
-	if e.ReplicationStats != nil {
-		// Clone ReplicationStats
-		e.ReplicationStats = e.ReplicationStats.clone()
-	}
+
 	if e.AllTierStats != nil {
 		e.AllTierStats = e.AllTierStats.clone()
 	}
@@ -931,22 +791,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 			ObjectSizesHistogram:    flat.ObjSizes.toMap(),
 			ObjectVersionsHistogram: flat.ObjVersions.toMap(),
 		}
-		if flat.ReplicationStats != nil {
-			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
-			bui.ReplicaCount = flat.ReplicationStats.ReplicaCount
-
-			bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
-			for arn, stat := range flat.ReplicationStats.Targets {
-				bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
-					ReplicationPendingSize:  stat.PendingSize,
-					ReplicatedSize:          stat.ReplicatedSize,
-					ReplicationFailedSize:   stat.FailedSize,
-					ReplicationPendingCount: stat.PendingCount,
-					ReplicationFailedCount:  stat.FailedCount,
-					ReplicatedCount:         stat.ReplicatedCount,
-				}
-			}
-		}
 		dst[bucket.Name] = bui
 	}
 	return dst
@@ -959,9 +803,6 @@ func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
 		return root
 	}
 	flat := d.flatten(*root)
-	if flat.ReplicationStats.empty() {
-		flat.ReplicationStats = nil
-	}
 	return &flat
 }
 
@@ -1238,20 +1079,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			if v.ReplicatedSize > 0 || v.ReplicaSize > 0 || v.ReplicationFailedSize > 0 || v.ReplicationPendingSize > 0 {
-				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
-				if cfg != nil && cfg.RoleArn != "" {
-					due.ReplicationStats = &replicationAllStats{
-						Targets: make(map[string]replicationStats),
-					}
-					due.ReplicationStats.ReplicaSize = v.ReplicaSize
-					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
-						ReplicatedSize: v.ReplicatedSize,
-						FailedSize:     v.ReplicationFailedSize,
-						PendingSize:    v.ReplicationPendingSize,
-					}
-				}
-			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
@@ -1277,36 +1104,10 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			empty := replicationStatsV1{}
-
-			if v.ReplicationStats != empty {
-				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
-				if cfg != nil && cfg.RoleArn != "" {
-					due.ReplicationStats = &replicationAllStats{
-						Targets: make(map[string]replicationStats),
-					}
-					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
-						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
-						FailedSize:     v.ReplicationStats.FailedSize,
-						FailedCount:    v.ReplicationStats.FailedCount,
-						PendingSize:    v.ReplicationStats.PendingSize,
-						PendingCount:   v.ReplicationStats.PendingCount,
-					}
-					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
-				}
-			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
 		}
-
-		// Populate compacted value and remove unneeded replica stats.
-		for k, e := range d.Cache {
-			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
-				e.ReplicationStats = nil
-			}
-			d.Cache[k] = e
-		}
 		return nil
 	case dataUsageCacheVerV5:
 		// Zstd compressed.
@@ -1328,36 +1129,10 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			if v.ReplicationStats != nil && !v.ReplicationStats.Empty() {
-				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
-				if cfg != nil && cfg.RoleArn != "" {
-					due.ReplicationStats = &replicationAllStats{
-						Targets: make(map[string]replicationStats),
-					}
-					d.Info.replication = replicationConfig{Config: cfg}
-
-					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
-						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
-						FailedSize:     v.ReplicationStats.FailedSize,
-						FailedCount:    v.ReplicationStats.FailedCount,
-						PendingSize:    v.ReplicationStats.PendingSize,
-						PendingCount:   v.ReplicationStats.PendingCount,
-					}
-					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
-				}
-			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
 		}
-
-		// Populate compacted value and remove unneeded replica stats.
-		for k, e := range d.Cache {
-			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
-				e.ReplicationStats = nil
-			}
-			d.Cache[k] = e
-		}
 		return nil
 	case dataUsageCacheVerV6:
 		// Zstd compressed.
@@ -1373,22 +1148,13 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		d.Info = dold.Info
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
-			var replicationStats *replicationAllStats
-			if v.ReplicationStats != nil {
-				replicationStats = &replicationAllStats{
-					Targets:      v.ReplicationStats.Targets,
-					ReplicaSize:  v.ReplicationStats.ReplicaSize,
-					ReplicaCount: v.ReplicationStats.ReplicaCount,
-				}
-			}
 			due := dataUsageEntry{
-				Children:         v.Children,
-				Size:             v.Size,
-				Objects:          v.Objects,
-				Versions:         v.Versions,
-				ObjSizes:         v.ObjSizes,
-				ReplicationStats: replicationStats,
-				Compacted:        v.Compacted,
+				Children:  v.Children,
+				Size:      v.Size,
+				Objects:   v.Objects,
+				Versions:  v.Versions,
+				ObjSizes:  v.ObjSizes,
+				Compacted: v.Compacted,
 			}
 			d.Cache[k] = due
 		}
@@ -1410,13 +1176,12 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			var szHist sizeHistogram
 			szHist.mergeV1(v.ObjSizes)
 			d.Cache[k] = dataUsageEntry{
-				Children:         v.Children,
-				Size:             v.Size,
-				Objects:          v.Objects,
-				Versions:         v.Versions,
-				ObjSizes:         szHist,
-				ReplicationStats: v.ReplicationStats,
-				Compacted:        v.Compacted,
+				Children:  v.Children,
+				Size:      v.Size,
+				Objects:   v.Objects,
+				Versions:  v.Versions,
+				ObjSizes:  szHist,
+				Compacted: v.Compacted,
 			}
 		}
 

cmd/data-usage-cache_gen_test.go

@@ -519,458 +519,6 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalreplicationAllStats(t *testing.T) {
-	v := replicationAllStats{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationAllStats(t *testing.T) {
-	v := replicationAllStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationAllStats Msgsize() is inaccurate")
-	}
-
-	vn := replicationAllStats{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationAllStats(b *testing.B) {
-	v := replicationAllStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalreplicationAllStatsV1(t *testing.T) {
-	v := replicationAllStatsV1{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationAllStatsV1(t *testing.T) {
-	v := replicationAllStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationAllStatsV1 Msgsize() is inaccurate")
-	}
-
-	vn := replicationAllStatsV1{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationAllStatsV1(b *testing.B) {
-	v := replicationAllStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalreplicationStats(t *testing.T) {
-	v := replicationStats{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationStats(b *testing.B) {
-	v := replicationStats{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationStats(b *testing.B) {
-	v := replicationStats{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationStats(b *testing.B) {
-	v := replicationStats{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationStats(t *testing.T) {
-	v := replicationStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationStats Msgsize() is inaccurate")
-	}
-
-	vn := replicationStats{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationStats(b *testing.B) {
-	v := replicationStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationStats(b *testing.B) {
-	v := replicationStats{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestMarshalUnmarshalreplicationStatsV1(t *testing.T) {
-	v := replicationStatsV1{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgreplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgreplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalreplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodereplicationStatsV1(t *testing.T) {
-	v := replicationStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodereplicationStatsV1 Msgsize() is inaccurate")
-	}
-
-	vn := replicationStatsV1{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodereplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodereplicationStatsV1(b *testing.B) {
-	v := replicationStatsV1{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
 	v := sizeHistogram{}
 	bts, err := v.MarshalMsg(nil)

cmd/data-usage_test.go

@@ -587,17 +587,6 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		t.Fatal(err)
 	}
 	e := want.find("abucket/dir2")
-	e.ReplicationStats = &replicationAllStats{
-		Targets: map[string]replicationStats{
-			"arn": {
-				PendingSize:    1,
-				ReplicatedSize: 2,
-				FailedSize:     3,
-				FailedCount:    5,
-				PendingCount:   6,
-			},
-		},
-	}
 	want.replace("abucket/dir2", "", *e)
 	var buf bytes.Buffer
 	err = want.serializeTo(&buf)

cmd/dynamic-timeouts.go

@@ -117,12 +117,12 @@ func (dt *dynamicTimeout) logEntry(duration time.Duration) {
 // adjust changes the value of the dynamic timeout based on the
 // previous results
 func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
-	failures, max := 0, time.Duration(0)
+	failures, maxDur := 0, time.Duration(0)
 	for _, dur := range entries[:] {
 		if dur == maxDuration {
 			failures++
-		} else if dur > max {
-			max = dur
+		} else if dur > maxDur {
+			maxDur = dur
 		}
 	}
 
@@ -144,12 +144,12 @@ func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
 	} else if failPct < dynamicTimeoutDecreaseThresholdPct {
 		// We are hitting the timeout relatively few times,
 		// so decrease the timeout towards 25 % of maximum time spent.
-		max = max * 125 / 100
+		maxDur = maxDur * 125 / 100
 
 		timeout := atomic.LoadInt64(&dt.timeout)
-		if max < time.Duration(timeout) {
+		if maxDur < time.Duration(timeout) {
 			// Move 50% toward the max.
-			timeout = (int64(max) + timeout) / 2
+			timeout = (int64(maxDur) + timeout) / 2
 		}
 		if timeout < dt.minimum {
 			timeout = dt.minimum

cmd/encryption-v1.go

@@ -109,7 +109,7 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
 // be AWS S3 compliant.
 //
 // DecryptETags uses a KMS bulk decryption API, if available, which
-// is more efficient than decrypting ETags sequentually.
+// is more efficient than decrypting ETags sequentially.
 func DecryptETags(ctx context.Context, k *kms.KMS, objects []ObjectInfo) error {
 	const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
 	var (
@@ -1099,7 +1099,9 @@ func (o *ObjectInfo) decryptPartsChecksums(h http.Header) {
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {
-			encLogIf(GlobalContext, err)
+			if !errors.Is(err, crypto.ErrSecretKeyMismatch) {
+				encLogIf(GlobalContext, err)
+			}
 			return
 		}
 		data = decrypted

cmd/endpoint.go

@@ -26,6 +26,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -38,7 +39,6 @@ import (
 	"github.com/minio/minio/internal/mountinfo"
 	"github.com/minio/pkg/v3/env"
 	xnet "github.com/minio/pkg/v3/net"
-	"golang.org/x/exp/slices"
 )
 
 // EndpointType - enum for endpoint type.

cmd/erasure-healing-common.go

@@ -277,23 +277,21 @@ func partNeedsHealing(partErrs []int) bool {
 	return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess && i != checkPartUnknown }) > -1
 }
 
-func hasPartErr(partErrs []int) bool {
-	return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess }) > -1
+func countPartNotSuccess(partErrs []int) (c int) {
+	for _, pe := range partErrs {
+		if pe != checkPartSuccess {
+			c++
+		}
+	}
+	return
 }
 
-// disksWithAllParts - This function needs to be called with
-// []StorageAPI returned by listOnlineDisks. Returns,
-//
-// - disks which have all parts specified in the latest xl.meta.
-//
-//   - slice of errors about the state of data files on disk - can have
-//     a not-found error or a hash-mismatch error.
-func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
-	errs []error, latestMeta FileInfo, bucket, object string,
+// checkObjectWithAllParts sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated
+// it also checks if the status of each part (corrupted, missing, ok) in each drive
+func checkObjectWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
+	errs []error, latestMeta FileInfo, filterByETag bool, bucket, object string,
 	scanMode madmin.HealScanMode,
-) (availableDisks []StorageAPI, dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
-	availableDisks = make([]StorageAPI, len(onlineDisks))
-
+) (dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
 	dataErrsByDisk = make(map[int][]int, len(onlineDisks))
 	for i := range onlineDisks {
 		dataErrsByDisk[i] = make([]int, len(latestMeta.Parts))
@@ -334,20 +332,28 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 
 	metaErrs := make([]error, len(errs))
 
-	for i, onlineDisk := range onlineDisks {
+	for i := range onlineDisks {
 		if errs[i] != nil {
 			metaErrs[i] = errs[i]
 			continue
 		}
-		if onlineDisk == OfflineDisk {
+		if onlineDisks[i] == OfflineDisk {
 			metaErrs[i] = errDiskNotFound
 			continue
 		}
 
 		meta := partsMetadata[i]
-		if !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir {
+		corrupted := false
+		if filterByETag {
+			corrupted = meta.Metadata["etag"] != latestMeta.Metadata["etag"]
+		} else {
+			corrupted = !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir
+		}
+
+		if corrupted {
 			metaErrs[i] = errFileCorrupt
 			partsMetadata[i] = FileInfo{}
+			onlineDisks[i] = nil
 			continue
 		}
 
@@ -355,6 +361,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			if !meta.IsValid() {
 				partsMetadata[i] = FileInfo{}
 				metaErrs[i] = errFileCorrupt
+				onlineDisks[i] = nil
 				continue
 			}
 
@@ -365,6 +372,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 					// might have the right erasure distribution.
 					partsMetadata[i] = FileInfo{}
 					metaErrs[i] = errFileCorrupt
+					onlineDisks[i] = nil
 					continue
 				}
 			}
@@ -385,8 +393,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		if metaErrs[i] != nil {
 			continue
 		}
-		meta := partsMetadata[i]
 
+		meta := partsMetadata[i]
 		if meta.Deleted || meta.IsRemote() {
 			continue
 		}
@@ -408,7 +416,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			verifyResp *CheckPartsResp
 		)
 
-		meta.DataDir = latestMeta.DataDir
 		switch scanMode {
 		case madmin.HealDeepScan:
 			// disk has a valid xl.meta but may not have all the
@@ -434,16 +441,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			dataErrsByDisk[disk][part] = dataErrsByPart[part][disk]
 		}
 	}
-
-	for i, onlineDisk := range onlineDisks {
-		if metaErrs[i] == nil && !hasPartErr(dataErrsByDisk[i]) {
-			// All parts verified, mark it as all data available.
-			availableDisks[i] = onlineDisk
-		} else {
-			// upon errors just make that disk's fileinfo invalid
-			partsMetadata[i] = FileInfo{}
-		}
-	}
-
 	return
 }

cmd/erasure-healing-common_test.go

@@ -313,11 +313,11 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
 			}
-			availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
-				test.errs, fi, bucket, object, madmin.HealDeepScan)
+			_, _ = checkObjectWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 			if test._tamperBackend != noTamper {
-				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
+				if tamperedIndex != -1 && onlineDisks[tamperedIndex] != nil {
 					t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
 						erasureDisks[tamperedIndex])
 				}
@@ -495,11 +495,11 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					test.expectedTime, modTime)
 			}
 
-			availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
-				test.errs, fi, bucket, object, madmin.HealDeepScan)
+			_, _ = checkObjectWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, false, bucket, object, madmin.HealDeepScan)
 
 			if test._tamperBackend != noTamper {
-				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
+				if tamperedIndex != -1 && onlineDisks[tamperedIndex] != nil {
 					t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
 						erasureDisks[tamperedIndex])
 				}
@@ -545,6 +545,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	// Test 1: Test that all disks are returned without any failures with
 	// unmodified meta data
+	erasureDisks = s.getDisks()
 	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	if err != nil {
 		t.Fatalf("Failed to read xl meta data %v", err)
@@ -557,14 +558,10 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)
 
-	filteredDisks, _, dataErrsPerDisk := disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+	dataErrsPerDisk, _ := checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
-	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
-	}
-
-	for diskIndex, disk := range filteredDisks {
+	for diskIndex, disk := range erasureDisks {
 		if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
 			t.Errorf("Unexpected error: %v", dataErrsPerDisk[diskIndex])
 		}
@@ -575,17 +572,15 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	// Test 2: Not synchronized modtime
+	erasureDisks = s.getDisks()
 	partsMetadataBackup := partsMetadata[0]
 	partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour)
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+	_, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
-	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
-	}
-	for diskIndex, disk := range filteredDisks {
+	for diskIndex, disk := range erasureDisks {
 		if diskIndex == 0 && disk != nil {
 			t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 		}
@@ -596,17 +591,15 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[0] = partsMetadataBackup // Revert before going to the next test
 
 	// Test 3: Not synchronized DataDir
+	erasureDisks = s.getDisks()
 	partsMetadataBackup = partsMetadata[1]
 	partsMetadata[1].DataDir = "foo-random"
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+	_, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
-	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
-	}
-	for diskIndex, disk := range filteredDisks {
+	for diskIndex, disk := range erasureDisks {
 		if diskIndex == 1 && disk != nil {
 			t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 		}
@@ -617,6 +610,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[1] = partsMetadataBackup // Revert before going to the next test
 
 	// Test 4: key = disk index, value = part name with hash mismatch
+	erasureDisks = s.getDisks()
 	diskFailures := make(map[int]string)
 	diskFailures[0] = "part.1"
 	diskFailures[3] = "part.1"
@@ -637,29 +631,18 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, dataErrsPerDisk, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
-		errs, fi, bucket, object, madmin.HealDeepScan)
+	dataErrsPerDisk, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, false, bucket, object, madmin.HealDeepScan)
 
-	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
-	}
-
-	for diskIndex, disk := range filteredDisks {
+	for diskIndex := range erasureDisks {
 		if _, ok := diskFailures[diskIndex]; ok {
-			if disk != nil {
-				t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
-			}
 			if !partNeedsHealing(dataErrsPerDisk[diskIndex]) {
 				t.Errorf("Disk expected to be healed, driveIndex: %d", diskIndex)
 			}
 		} else {
-			if disk == nil {
-				t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
-			}
 			if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
 				t.Errorf("Disk not expected to be healed, driveIndex: %d", diskIndex)
 			}
-
 		}
 	}
 }

cmd/erasure-healing.go

@@ -32,7 +32,6 @@ import (
 	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/sync/errgroup"
-	"golang.org/x/exp/slices"
 )
 
 //go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
@@ -45,7 +44,8 @@ const (
 	healingMetricCheckAbandonedParts
 )
 
-func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
+// List a prefix or a single object versions and heal
+func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string, recursive bool, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -78,11 +78,14 @@ func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string,
 		bucket:         bucket,
 		path:           path,
 		filterPrefix:   filterPrefix,
-		recursive:      true,
+		recursive:      recursive,
 		forwardTo:      "",
 		minDisks:       1,
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
+			if !recursive && prefix != entry.name {
+				return
+			}
 			if err := healEntry(bucket, entry, scanMode); err != nil {
 				cancel()
 			}
@@ -94,7 +97,9 @@ func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string,
 				// proceed to heal nonetheless.
 				entry, _ = entries.firstFound()
 			}
-
+			if !recursive && prefix != entry.name {
+				return
+			}
 			if err := healEntry(bucket, *entry, scanMode); err != nil {
 				cancel()
 				return
@@ -149,37 +154,40 @@ var errLegacyXLMeta = errors.New("legacy XL meta")
 
 var errOutdatedXLMeta = errors.New("outdated XL meta")
 
-var errPartMissingOrCorrupt = errors.New("part missing or corrupt")
+var (
+	errPartCorrupt = errors.New("part corrupt")
+	errPartMissing = errors.New("part missing")
+)
 
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, error) {
+func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, bool, error) {
 	if errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound) || errors.Is(erErr, errFileCorrupt) {
-		return true, erErr
+		return true, true, erErr
 	}
 	if erErr == nil {
 		if meta.XLV1 {
 			// Legacy means heal always
 			// always check first.
-			return true, errLegacyXLMeta
+			return true, true, errLegacyXLMeta
 		}
 		if !latestMeta.Equals(meta) {
-			return true, errOutdatedXLMeta
+			return true, true, errOutdatedXLMeta
 		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			for _, partErr := range partsErrs {
-				if slices.Contains([]int{
-					checkPartFileNotFound,
-					checkPartFileCorrupt,
-				}, partErr) {
-					return true, errPartMissingOrCorrupt
+				if partErr == checkPartFileNotFound {
+					return true, false, errPartMissing
+				}
+				if partErr == checkPartFileCorrupt {
+					return true, false, errPartCorrupt
 				}
 			}
 		}
-		return false, nil
+		return false, false, nil
 	}
-	return false, erErr
+	return false, false, erErr
 }
 
 const (
@@ -234,11 +242,12 @@ func (er *erasureObjects) auditHealObject(ctx context.Context, bucket, object, v
 	}
 
 	b, a := result.GetCorruptedCounts()
-	if b == a {
+	if b > 0 && b == a {
 		opts.Error = fmt.Sprintf("unable to heal %d corrupted blocks on drives", b)
 	}
+
 	b, a = result.GetMissingCounts()
-	if b == a {
+	if b > 0 && b == a {
 		opts.Error = fmt.Sprintf("unable to heal %d missing blocks on drives", b)
 	}
 
@@ -253,6 +262,21 @@ func (er *erasureObjects) auditHealObject(ctx context.Context, bucket, object, v
 	auditLogInternal(ctx, opts)
 }
 
+func objectErrToDriveState(reason error) string {
+	switch {
+	case reason == nil:
+		return madmin.DriveStateOk
+	case IsErr(reason, errDiskNotFound):
+		return madmin.DriveStateOffline
+	case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissing, errOutdatedXLMeta, errLegacyXLMeta):
+		return madmin.DriveStateMissing
+	case IsErr(reason, errFileCorrupt, errPartCorrupt):
+		return madmin.DriveStateCorrupt
+	default:
+		return fmt.Sprintf("%s (%s)", madmin.DriveStateUnknown, reason.Error())
+	}
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -327,26 +351,20 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 	// List of disks having latest version of the object xl.meta
 	// (by modtime).
-	onlineDisks, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum)
+	onlineDisks, quorumModTime, quorumETag := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum)
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, etag, readQuorum)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, quorumModTime, quorumETag, readQuorum)
 	if err != nil {
 		return result, err
 	}
 
-	// List of disks having all parts as per latest metadata.
-	// NOTE: do not pass in latestDisks to diskWithAllParts since
-	// the diskWithAllParts needs to reach the drive to ensure
-	// validity of the metadata content, we should make sure that
-	// we pass in disks as is for it to be verified. Once verified
-	// the disksWithAllParts() returns the actual disks that can be
-	// used here for reconstruction. This is done to ensure that
-	// we do not skip drives that have inconsistent metadata to be
-	// skipped from purging when they are stale.
-	availableDisks, dataErrsByDisk, dataErrsByPart := disksWithAllParts(ctx, onlineDisks, partsMetadata,
-		errs, latestMeta, bucket, object, scanMode)
+	// No modtime quorum
+	filterDisksByETag := quorumETag != ""
+
+	dataErrsByDisk, dataErrsByPart := checkObjectWithAllParts(ctx, onlineDisks, partsMetadata,
+		errs, latestMeta, filterDisksByETag, bucket, object, scanMode)
 
 	var erasure Erasure
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
@@ -367,26 +385,18 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
 	outDatedDisks := make([]StorageAPI, len(storageDisks))
-	disksToHealCount := 0
-	for i := range availableDisks {
-		yes, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta)
+	disksToHealCount, xlMetaToHealCount := 0, 0
+	for i := range onlineDisks {
+		yes, isMeta, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta)
 		if yes {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
+			if isMeta {
+				xlMetaToHealCount++
+			}
 		}
 
-		driveState := ""
-		switch {
-		case reason == nil:
-			driveState = madmin.DriveStateOk
-		case IsErr(reason, errDiskNotFound):
-			driveState = madmin.DriveStateOffline
-		case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissingOrCorrupt, errOutdatedXLMeta, errLegacyXLMeta):
-			driveState = madmin.DriveStateMissing
-		default:
-			// all remaining cases imply corrupt data/metadata
-			driveState = madmin.DriveStateCorrupt
-		}
+		driveState := objectErrToDriveState(reason)
 
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
@@ -400,16 +410,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		})
 	}
 
-	if isAllNotFound(errs) {
-		// File is fully gone, fileInfo is empty.
-		err := errFileNotFound
-		if versionID != "" {
-			err = errFileVersionNotFound
-		}
-		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), err
-	}
-
 	if disksToHealCount == 0 {
 		// Nothing to heal!
 		return result, nil
@@ -421,7 +421,24 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	cannotHeal := !latestMeta.XLV1 && !latestMeta.Deleted && xlMetaToHealCount > latestMeta.Erasure.ParityBlocks
+	if cannotHeal && quorumETag != "" {
+		// This is an object that is supposed to be removed by the dangling code
+		// but we noticed that ETag is the same for all objects, let's give it a shot
+		cannotHeal = false
+	}
+
+	if !latestMeta.Deleted && !latestMeta.IsRemote() {
+		// check if there is a part that lost its quorum
+		for _, partErrs := range dataErrsByPart {
+			if countPartNotSuccess(partErrs) > latestMeta.Erasure.ParityBlocks {
+				cannotHeal = true
+				break
+			}
+		}
+	}
+
+	if cannotHeal {
 		// Allow for dangling deletes, on versions that have DataDir missing etc.
 		// this would end up restoring the correct readable versions.
 		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrsByPart, ObjectOptions{
@@ -459,16 +476,15 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	tmpID := mustGetUUID()
 	migrateDataDir := mustGetUUID()
 
-	// Reorder so that we have data disks first and parity disks next.
-	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
-		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
-			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
-		healingLogOnceIf(ctx, err, "heal-object-available-disks")
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(onlineDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, onlineDisks, bucket, object, versionID)
+		healingLogOnceIf(ctx, err, "heal-object-online-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
 
-	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+	latestDisks := shuffleDisks(onlineDisks, latestMeta.Erasure.Distribution)
 
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
@@ -539,6 +555,10 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 				if disk == OfflineDisk {
 					continue
 				}
+				thisPartErrs := shuffleCheckParts(dataErrsByPart[partIndex], latestMeta.Erasure.Distribution)
+				if thisPartErrs[i] != checkPartSuccess {
+					continue
+				}
 				checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber)
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
 				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
@@ -806,13 +826,7 @@ func (er *erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []Storage
 			})
 			continue
 		}
-		driveState := madmin.DriveStateCorrupt
-		switch errs[index] {
-		case errFileNotFound, errVolumeNotFound:
-			driveState = madmin.DriveStateMissing
-		case nil:
-			driveState = madmin.DriveStateOk
-		}
+		driveState := objectErrToDriveState(errs[index])
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
 			Endpoint: storageEndpoints[index].String(),

cmd/erasure-healing_test.go

@@ -733,7 +733,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions)
 	}
 
-	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
+	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true},
 		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
 			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
@@ -780,7 +780,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions)
 	}
 
-	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
+	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true},
 		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
 			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
@@ -829,7 +829,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatalf("Expected versions 3, got %d", fileInfoPreHeal.NumVersions)
 	}
 
-	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
+	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true},
 		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
 			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err

cmd/erasure-metadata-utils.go

@@ -52,15 +52,15 @@ func reduceCommonVersions(diskVersions [][]byte, writeQuorum int) (versions []by
 	}
 
 	var commonVersions uint64
-	max := 0
+	maxCnt := 0
 	for versions, count := range diskVersionsCount {
-		if max < count {
-			max = count
+		if maxCnt < count {
+			maxCnt = count
 			commonVersions = versions
 		}
 	}
 
-	if max >= writeQuorum {
+	if maxCnt >= writeQuorum {
 		for _, versions := range diskVersions {
 			if binary.BigEndian.Uint64(versions) == commonVersions {
 				return versions
@@ -80,15 +80,15 @@ func reduceCommonDataDir(dataDirs []string, writeQuorum int) (dataDir string) {
 		dataDirsCount[ddir]++
 	}
 
-	max := 0
+	maxCnt := 0
 	for ddir, count := range dataDirsCount {
-		if max < count {
-			max = count
+		if maxCnt < count {
+			maxCnt = count
 			dataDir = ddir
 		}
 	}
 
-	if max >= writeQuorum {
+	if maxCnt >= writeQuorum {
 		return dataDir
 	}
 
@@ -115,20 +115,20 @@ func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error)
 		errorCounts[err]++
 	}
 
-	max := 0
+	maxCnt := 0
 	for err, count := range errorCounts {
 		switch {
-		case max < count:
-			max = count
+		case maxCnt < count:
+			maxCnt = count
 			maxErr = err
 
 		// Prefer `nil` over other error values with the same
 		// number of occurrences.
-		case max == count && err == nil:
+		case maxCnt == count && err == nil:
 			maxErr = err
 		}
 	}
-	return max, maxErr
+	return maxCnt, maxErr
 }
 
 // reduceQuorumErrs behaves like reduceErrs by only for returning
@@ -295,34 +295,34 @@ func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo,
 	return shuffledDisks, shuffledPartsMetadata
 }
 
-// Return shuffled partsMetadata depending on distribution.
-func shufflePartsMetadata(partsMetadata []FileInfo, distribution []int) (shuffledPartsMetadata []FileInfo) {
+func shuffleWithDist[T any](input []T, distribution []int) []T {
 	if distribution == nil {
-		return partsMetadata
+		return input
 	}
-	shuffledPartsMetadata = make([]FileInfo, len(partsMetadata))
-	// Shuffle slice xl metadata for expected distribution.
-	for index := range partsMetadata {
+	shuffled := make([]T, len(input))
+	for index := range input {
 		blockIndex := distribution[index]
-		shuffledPartsMetadata[blockIndex-1] = partsMetadata[index]
+		shuffled[blockIndex-1] = input[index]
 	}
-	return shuffledPartsMetadata
+	return shuffled
+}
+
+// Return shuffled partsMetadata depending on distribution.
+func shufflePartsMetadata(partsMetadata []FileInfo, distribution []int) []FileInfo {
+	return shuffleWithDist[FileInfo](partsMetadata, distribution)
+}
+
+// shuffleCheckParts - shuffle CheckParts slice depending on the
+// erasure distribution.
+func shuffleCheckParts(parts []int, distribution []int) []int {
+	return shuffleWithDist[int](parts, distribution)
 }
 
 // shuffleDisks - shuffle input disks slice depending on the
 // erasure distribution. Return shuffled slice of disks with
 // their expected distribution.
-func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []StorageAPI) {
-	if distribution == nil {
-		return disks
-	}
-	shuffledDisks = make([]StorageAPI, len(disks))
-	// Shuffle disks for expected distribution.
-	for index := range disks {
-		blockIndex := distribution[index]
-		shuffledDisks[blockIndex-1] = disks[index]
-	}
-	return shuffledDisks
+func shuffleDisks(disks []StorageAPI, distribution []int) []StorageAPI {
+	return shuffleWithDist[StorageAPI](disks, distribution)
 }
 
 // evalDisks - returns a new slice of disks where nil is set if

cmd/erasure-metadata.go

@@ -514,8 +514,10 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 		if metadata.Deleted || metadata.Size == 0 {
 			parities[index] = totalShards / 2
 		} else if metadata.TransitionStatus == lifecycle.TransitionComplete {
-			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. It is not equal to EcM because the data integrity is entrusted with the warm tier.
-			parities[index] = totalShards - (totalShards/2 + 1)
+			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta.
+			// It is not equal to EcM because the data integrity is entrusted with the warm tier.
+			// However, we never go below EcM, in case of a EcM=EcN setup.
+			parities[index] = max(totalShards-(totalShards/2+1), metadata.Erasure.ParityBlocks)
 		} else {
 			parities[index] = metadata.Erasure.ParityBlocks
 		}

cmd/erasure-multipart.go

@@ -526,6 +526,14 @@ func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object
 
 // renamePart - renames multipart part to its relevant location under uploadID.
 func (er erasureObjects) renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, optsMeta []byte, writeQuorum int) ([]StorageAPI, error) {
+	paths := []string{
+		dstEntry,
+		dstEntry + ".meta",
+	}
+
+	// cleanup existing paths first across all drives.
+	er.cleanupMultipartPath(ctx, paths...)
+
 	g := errgroup.WithNErrs(len(disks))
 
 	// Rename file on all underlying storage disks.
@@ -542,11 +550,6 @@ func (er erasureObjects) renamePart(ctx context.Context, disks []StorageAPI, src
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
-	paths := []string{
-		dstEntry,
-		dstEntry + ".meta",
-	}
-
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if err != nil {
 		er.cleanupMultipartPath(ctx, paths...)
@@ -574,19 +577,9 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(errInvalidArgument)
 	}
 
-	// Read lock for upload id.
-	// Only held while reading the upload metadata.
-	uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return PartInfo{}, err
-	}
-	rctx := rlkctx.Context()
-	defer uploadIDRLock.RUnlock(rlkctx)
-
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	// Validates if upload ID exists.
-	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
+	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
 			return pi, toObjectErr(err, bucket)
@@ -741,10 +734,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
-	// Write lock for this part ID, only hold it if we are planning to read from the
-	// stream avoid any concurrent updates.
-	//
-	// Must be held throughout this call.
+	// Serialize concurrent part uploads.
 	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
 	plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
@@ -754,6 +744,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	ctx = plkctx.Context()
 	defer partIDLock.Unlock(plkctx)
 
+	// Read lock for upload id, only held while reading the upload metadata.
+	uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return PartInfo{}, err
+	}
+	ctx = rlkctx.Context()
+	defer uploadIDRLock.RUnlock(rlkctx)
+
 	onlineDisks, err = er.renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, partFI, writeQuorum)
 	if err != nil {
 		if errors.Is(err, errFileNotFound) {
@@ -798,14 +797,6 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
-	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return MultipartInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer uploadIDLock.RUnlock(lkctx)
-
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
@@ -885,14 +876,6 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		auditObjectErasureSet(ctx, "ListObjectParts", object, &er)
 	}
 
-	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return ListPartsInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer uploadIDLock.RUnlock(lkctx)
-
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
@@ -1115,16 +1098,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
-	// Hold write locks to verify uploaded parts, also disallows any
-	// parallel PutObjectPart() requests.
-	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return oi, err
-	}
-	ctx = wlkctx.Context()
-	defer uploadIDLock.Unlock(wlkctx)
-
 	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
@@ -1491,14 +1464,6 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 		auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
 	}
 
-	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx)
-
 	// Validates if upload ID exists.
 	if _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false); err != nil {
 		if errors.Is(err, errVolumeNotFound) {

cmd/erasure-object.go

@@ -102,7 +102,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(srcBucket, minioMetaBucket) {
+		if shouldCheckForDangling(err, errs, srcBucket) {
 			_, derr := er.deleteIfDangling(context.Background(), srcBucket, srcObject, metaArr, errs, nil, srcOpts)
 			if derr == nil {
 				if srcOpts.VersionID != "" {
@@ -690,13 +690,9 @@ func shouldCheckForDangling(err error, errs []error, bucket string) bool {
 	// Check if we have a read quorum issue
 	case errors.Is(err, errErasureReadQuorum):
 		return true
-	// Check if the object is inexistent in most disks but not all of them
-	case errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound):
-		for i := range errs {
-			if errs[i] == nil {
-				return true
-			}
-		}
+	// Check if the object is non-existent on most disks but not all of them
+	case (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) && (countErrs(errs, nil) > 0):
+		return true
 	}
 	return false
 }
@@ -921,6 +917,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 				}
 			}
 		}
+		// when we have insufficient read quorum and inconsistent metadata return
+		// file not found, since we can't possibly have a way to recover this object
+		// anyway.
+		if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta {
+			if opts.VersionID != "" {
+				err = errFileVersionNotFound
+			} else {
+				err = errFileNotFound
+			}
+		}
 		return fi, nil, nil, toObjectErr(err, bucket, object)
 	}
 
@@ -1329,10 +1335,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.DataDir = mustGetUUID()
-	fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
-	if opts.EncryptFn != nil {
-		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
-	}
 	if ckSum := userDefined[ReplicationSsecChecksumHeader]; ckSum != "" {
 		if v, err := base64.StdEncoding.DecodeString(ckSum); err == nil {
 			fi.Checksum = v
@@ -1460,7 +1462,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			actualSize = n
 		}
 	}
-
+	if fi.Checksum == nil {
+		// Trailing headers checksums should now be filled.
+		fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
+		if opts.EncryptFn != nil {
+			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
+		}
+	}
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -1474,6 +1482,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		// No need to add checksum to part. We already have it on the object.
 		partsMetadata[i].AddObjectPart(1, "", n, actualSize, modTime, compIndex, nil)
 		partsMetadata[i].Versioned = opts.Versioned || opts.VersionSuspended
+		partsMetadata[i].Checksum = fi.Checksum
 	}
 
 	userDefined["etag"] = r.MD5CurrentHexString()
@@ -2135,7 +2144,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+		if shouldCheckForDangling(err, errs, bucket) {
 			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr == nil {
 				if opts.VersionID != "" {
@@ -2189,14 +2198,16 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 // PutObjectTags - replace or add tags to an existing object
 func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) {
-	// Lock the object before updating tags.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
+	if !opts.NoLock {
+		// Lock the object before updating tags.
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
 	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx)
 
 	disks := er.getDisks()
 
@@ -2212,7 +2223,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+		if shouldCheckForDangling(err, errs, bucket) {
 			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr == nil {
 				if opts.VersionID != "" {
@@ -2307,14 +2318,16 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		return err
 	}
 
-	// Acquire write lock before starting to transition the object.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
-	if err != nil {
-		return err
+	if !opts.NoLock {
+		// Acquire write lock before starting to transition the object.
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
 	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx)
 
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
@@ -2352,6 +2365,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 
 	destObj, err := genTransitionObjName(bucket)
 	if err != nil {
+		traceFn(ILMTransition, nil, err)
 		return err
 	}
 
@@ -2362,9 +2376,14 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	}()
 
 	var rv remoteVersionID
-	rv, err = tgtClient.Put(ctx, destObj, pr, fi.Size)
+	rv, err = tgtClient.PutWithMeta(ctx, destObj, pr, fi.Size, map[string]string{
+		"name": object, // preserve the original name of the object on the remote tier object metadata.
+		// this is just for future reverse lookup() purposes (applies only for new objects)
+		// does not apply retro-actively on already transitioned objects.
+	})
 	pr.CloseWithError(err)
 	if err != nil {
+		traceFn(ILMTransition, nil, err)
 		return err
 	}
 	fi.TransitionStatus = lifecycle.TransitionComplete

cmd/erasure-server-pool.go

@@ -256,7 +256,7 @@ func (z *erasureServerPools) NewNSLock(bucket string, objects ...string) RWLocke
 
 // GetDisksID will return disks by their ID.
 func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
-	idMap := make(map[string]struct{})
+	idMap := make(map[string]struct{}, len(ids))
 	for _, id := range ids {
 		idMap[id] = struct{}{}
 	}
@@ -357,7 +357,7 @@ func (p serverPoolsAvailableSpace) TotalAvailable() uint64 {
 
 // FilterMaxUsed will filter out any pools that has used percent bigger than max,
 // unless all have that, in which case all are preserved.
-func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
+func (p serverPoolsAvailableSpace) FilterMaxUsed(maxUsed int) {
 	// We aren't modifying p, only entries in it, so we don't need to receive a pointer.
 	if len(p) <= 1 {
 		// Nothing to do.
@@ -365,7 +365,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
 	}
 	var ok bool
 	for _, z := range p {
-		if z.Available > 0 && z.MaxUsedPct < max {
+		if z.Available > 0 && z.MaxUsedPct < maxUsed {
 			ok = true
 			break
 		}
@@ -378,7 +378,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
 
 	// Remove entries that are above.
 	for i, z := range p {
-		if z.Available > 0 && z.MaxUsedPct < max {
+		if z.Available > 0 && z.MaxUsedPct < maxUsed {
 			continue
 		}
 		p[i].Available = 0
@@ -556,7 +556,7 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 			return pinfo, z.poolsWithObject(poolObjInfos, opts), nil
 		}
 		defPool = pinfo
-		if !isErrObjectNotFound(pinfo.Err) {
+		if !isErrObjectNotFound(pinfo.Err) && !isErrVersionNotFound(pinfo.Err) {
 			return pinfo, noReadQuorumPools, pinfo.Err
 		}
 
@@ -1166,13 +1166,34 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		}
 	}
 
+	if opts.DataMovement {
+		objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
+		objInfo.Name = decodeDirObject(object)
+		return objInfo, err
+	}
+
 	// Delete concurrently in all server pools with read quorum error for unversioned objects.
 	if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended {
 		return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools)
 	}
-	objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
+
+	for _, pool := range z.serverPools {
+		objInfo, err := pool.DeleteObject(ctx, bucket, object, opts)
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			objInfo.Name = decodeDirObject(object)
+			return objInfo, err
+		}
+		if err == nil {
+			objInfo.Name = decodeDirObject(object)
+			return objInfo, nil
+		}
+	}
+
 	objInfo.Name = decodeDirObject(object)
-	return objInfo, err
+	if opts.VersionID != "" {
+		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+	}
+	return objInfo, ObjectNotFound{Bucket: bucket, Object: object}
 }
 
 func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) {
@@ -1291,19 +1312,20 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 			go func(idx int, pool *erasureSets) {
 				defer wg.Done()
 				objs := poolObjIdxMap[idx]
-				if len(objs) > 0 {
-					orgIndexes := origIndexMap[idx]
-					deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
-					mu.Lock()
-					for i, derr := range errs {
-						if derr != nil {
-							derrs[orgIndexes[i]] = derr
-						}
-						deletedObjects[i].ObjectName = decodeDirObject(deletedObjects[i].ObjectName)
-						dobjects[orgIndexes[i]] = deletedObjects[i]
-					}
-					mu.Unlock()
+				if len(objs) == 0 {
+					return
 				}
+				orgIndexes := origIndexMap[idx]
+				deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
+				mu.Lock()
+				for i, derr := range errs {
+					if derr != nil {
+						derrs[orgIndexes[i]] = derr
+					}
+					deletedObjects[i].ObjectName = decodeDirObject(deletedObjects[i].ObjectName)
+					dobjects[orgIndexes[i]] = deletedObjects[i]
+				}
+				mu.Unlock()
 			}(idx, pool)
 		}
 		wg.Wait()
@@ -1868,9 +1890,18 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		return MultipartInfo{}, err
 	}
 
+	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return MultipartInfo{}, err
+	}
+	ctx = lkctx.Context()
+	defer uploadIDLock.RUnlock(lkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
+
 	for idx, pool := range z.serverPools {
 		if z.IsSuspended(idx) {
 			continue
@@ -1886,6 +1917,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		// any other unhandled error return right here.
 		return MultipartInfo{}, err
 	}
+
 	return MultipartInfo{}, InvalidUploadID{
 		Bucket:   bucket,
 		Object:   object,
@@ -1899,9 +1931,18 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		return ListPartsInfo{}, err
 	}
 
+	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return ListPartsInfo{}, err
+	}
+	ctx = lkctx.Context()
+	defer uploadIDLock.RUnlock(lkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
+
 	for idx, pool := range z.serverPools {
 		if z.IsSuspended(idx) {
 			continue
@@ -1915,6 +1956,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		}
 		return ListPartsInfo{}, err
 	}
+
 	return ListPartsInfo{}, InvalidUploadID{
 		Bucket:   bucket,
 		Object:   object,
@@ -1935,6 +1977,14 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		}
 	}()
 
+	lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
@@ -1973,6 +2023,16 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		}
 	}()
 
+	// Hold write locks to verify uploaded parts, also disallows any
+	// parallel PutObjectPart() requests.
+	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return objInfo, err
+	}
+	ctx = wlkctx.Context()
+	defer uploadIDLock.Unlock(wlkctx)
+
 	if z.SinglePool() {
 		return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
@@ -2408,6 +2468,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string, scanMode madmin.HealScanMode) error
 
+// List a prefix or a single object versions and heal
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
 	healEntry := func(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
 		if entry.isDir() {
@@ -2471,7 +2532,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
 
-				errs[idx] = set.listAndHeal(ctx, bucket, prefix, opts.ScanMode, healEntry)
+				errs[idx] = set.listAndHeal(ctx, bucket, prefix, opts.Recursive, opts.ScanMode, healEntry)
 			}(idx, set)
 		}
 		wg.Wait()
@@ -2523,15 +2584,22 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 		}
 	}
 
+	hr := madmin.HealResultItem{
+		Type:      madmin.HealItemObject,
+		Bucket:    bucket,
+		Object:    object,
+		VersionID: versionID,
+	}
+
 	// At this stage, all errors are 'not found'
 	if versionID != "" {
-		return madmin.HealResultItem{}, VersionNotFound{
+		return hr, VersionNotFound{
 			Bucket:    bucket,
 			Object:    object,
 			VersionID: versionID,
 		}
 	}
-	return madmin.HealResultItem{}, ObjectNotFound{
+	return hr, ObjectNotFound{
 		Bucket: bucket,
 		Object: object,
 	}
@@ -2752,7 +2820,19 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
 		return z.serverPools[0].PutObjectMetadata(ctx, bucket, object, opts)
 	}
 
+	if !opts.NoLock {
+		// Lock the object before updating metadata.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	opts.MetadataChg = true
+	opts.NoLock = true
 	// We don't know the size here set 1GiB at least.
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
@@ -2769,7 +2849,19 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
 
+	if !opts.NoLock {
+		// Lock the object before updating tags.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	opts.MetadataChg = true
+	opts.NoLock = true
 
 	// We don't know the size here set 1GiB at least.
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
@@ -2787,8 +2879,19 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 
-	opts.MetadataChg = true
+	if !opts.NoLock {
+		// Lock the object before deleting tags.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
 
+	opts.MetadataChg = true
+	opts.NoLock = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2819,8 +2922,20 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
+	if !opts.NoLock {
+		// Acquire write lock before starting to transition the object.
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	// Avoid transitioning an object from a pool being decommissioned.
 	opts.SkipDecommissioned = true
+	opts.NoLock = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
@@ -2836,8 +2951,20 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
+	if !opts.NoLock {
+		// Acquire write lock before restoring transitioned object
+		lk := z.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
 	// Avoid restoring object from a pool being decommissioned.
 	opts.SkipDecommissioned = true
+	opts.NoLock = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err

cmd/erasure-sets.go

@@ -86,6 +86,8 @@ type erasureSets struct {
 	lastConnectDisksOpTime time.Time
 }
 
+var staleUploadsCleanupIntervalChangedCh = make(chan struct{})
+
 func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 	diskMap := make(map[Endpoint]StorageAPI)
 
@@ -532,10 +534,11 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 				}(set)
 			}
 			wg.Wait()
-
-			// Reset for the next interval
-			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
+		case <-staleUploadsCleanupIntervalChangedCh:
 		}
+
+		// Reset for the next interval
+		timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
 	}
 }
 
@@ -567,10 +570,7 @@ func auditObjectErasureSet(ctx context.Context, api, object string, set *erasure
 
 // NewNSLock - initialize a new namespace RWLocker instance.
 func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
-	if len(objects) == 1 {
-		return s.getHashedSet(objects[0]).NewNSLock(bucket, objects...)
-	}
-	return s.getHashedSet("").NewNSLock(bucket, objects...)
+	return s.sets[0].NewNSLock(bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.

cmd/erasure.go

@@ -551,9 +551,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				var root dataUsageEntry
 				if r := cache.root(); r != nil {
 					root = cache.flatten(*r)
-					if root.ReplicationStats.empty() {
-						root.ReplicationStats = nil
-					}
 				}
 				select {
 				case <-ctx.Done():

cmd/ftp-server-driver.go

@@ -24,6 +24,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -286,6 +288,10 @@ func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string)
 }
 
 func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error) {
+	tr := http.RoundTripper(globalRemoteFTPClientTransport)
+	if host, _, err := net.SplitHostPort(ctx.Sess.RemoteAddr().String()); err == nil {
+		tr = forwardForTransport{tr: tr, fwd: host}
+	}
 	ui, ok := globalIAMSys.GetUser(context.Background(), ctx.Sess.LoginUser())
 	if !ok && !globalIAMSys.LDAPConfig.Enabled() {
 		return nil, errNoSuchUser
@@ -363,7 +369,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 		return minio.New(driver.endpoint, &minio.Options{
 			Creds:     mcreds,
 			Secure:    globalIsTLS,
-			Transport: globalRemoteFTPClientTransport,
+			Transport: tr,
 		})
 	}
 
@@ -377,7 +383,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 	return minio.New(driver.endpoint, &minio.Options{
 		Creds:     credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""),
 		Secure:    globalIsTLS,
-		Transport: globalRemoteFTPClientTransport,
+		Transport: tr,
 	})
 }
 

cmd/generic-handlers.go

@@ -33,8 +33,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/grid"
 	xnet "github.com/minio/pkg/v3/net"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/config/dns"
@@ -75,7 +73,7 @@ const (
 // and must not set by clients
 func containsReservedMetadata(header http.Header) bool {
 	for key := range header {
-		if slices.Contains(maps.Keys(validSSEReplicationHeaders), key) {
+		if _, ok := validSSEReplicationHeaders[key]; ok {
 			return false
 		}
 		if stringsHasPrefixFold(key, ReservedMetadataPrefix) {

cmd/global-heal.go

@@ -459,8 +459,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					continue
 				}
 
-				var versionHealed bool
-
 				res, err := er.HealObject(ctx, bucket, encodedEntryName,
 					version.VersionID, madmin.HealOpts{
 						ScanMode: scanMode,
@@ -475,21 +473,23 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					}
 				} else {
 					// Look for the healing results
-					if res.After.Drives[tracker.DiskIndex].State == madmin.DriveStateOk {
-						versionHealed = true
+					if res.After.Drives[tracker.DiskIndex].State != madmin.DriveStateOk {
+						err = fmt.Errorf("unexpected after heal state: %s", res.After.Drives[tracker.DiskIndex].State)
 					}
 				}
 
-				if versionHealed {
+				if err == nil {
 					bgSeq.countHealed(madmin.HealItemObject)
 					result = healEntrySuccess(uint64(version.Size))
 				} else {
 					bgSeq.countFailed(madmin.HealItemObject)
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
-						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s (version-id=%s): %w",
+							bucket, version.Name, version.VersionID, err))
 					} else {
-						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w",
+							bucket, version.Name, err))
 					}
 				}
 

cmd/handler-api.go

@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/shirou/gopsutil/v3/mem"
 
 	"github.com/minio/minio/internal/config/api"
@@ -61,7 +62,6 @@ type apiConfig struct {
 const (
 	cgroupV1MemLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
 	cgroupV2MemLimitFile = "/sys/fs/cgroup/memory.max"
-	cgroupMemNoLimit     = 9223372036854771712
 )
 
 func cgroupMemLimit() (limit uint64) {
@@ -78,10 +78,8 @@ func cgroupMemLimit() (limit uint64) {
 		// but still, no need to interpret more
 		return 0
 	}
-	if limit == cgroupMemNoLimit {
-		// No limit set, It's the highest positive signed 64-bit
-		// integer (2^63-1), rounded down to multiples of 4096 (2^12),
-		// the most common page size on x86 systems - for cgroup_limits.
+	if limit >= 100*humanize.TiByte {
+		// No limit set, or unreasonably high. Ignore
 		return 0
 	}
 	return limit
@@ -183,13 +181,22 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 	t.transitionWorkers = cfg.TransitionWorkers
 
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
-	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
 	t.enableODirect = cfg.EnableODirect
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
 	t.syncEvents = cfg.SyncEvents
 	t.objectMaxVersions = cfg.ObjectMaxVersions
+
+	if t.staleUploadsCleanupInterval != cfg.StaleUploadsCleanupInterval {
+		t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
+
+		// signal that cleanup interval has changed
+		select {
+		case staleUploadsCleanupIntervalChangedCh <- struct{}{}:
+		default: // in case the channel is blocked...
+		}
+	}
 }
 
 func (t *apiConfig) odirectEnabled() bool {

cmd/http-stats.go

@@ -133,7 +133,7 @@ func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.Response
 		bucketHTTPRequestsDuration.With(prometheus.Labels{
 			"api":    api,
 			"bucket": bucket,
-		}).Observe(w.TimeToFirstByte.Seconds())
+		}).Observe(w.TTFB().Seconds())
 	}
 
 	bh.Lock()
@@ -433,7 +433,7 @@ func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label
-	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
+	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TTFB().Seconds())
 
 	code := w.StatusCode
 

cmd/http-tracer.go

@@ -164,7 +164,7 @@ func httpTracerMiddleware(h http.Handler) http.Handler {
 					Latency:         reqEndTime.Sub(respRecorder.StartTime),
 					InputBytes:      inputBytes,
 					OutputBytes:     respRecorder.Size(),
-					TimeToFirstByte: respRecorder.TimeToFirstByte,
+					TimeToFirstByte: respRecorder.TTFB(),
 				},
 			},
 		}

cmd/iam-object-store.go

@@ -481,12 +481,24 @@ var (
 	policyDBGroupsListKey   = "policydb/groups/"
 )
 
+func findSecondIndex(s string, substr string) int {
+	first := strings.Index(s, substr)
+	if first == -1 {
+		return -1
+	}
+	second := strings.Index(s[first+1:], substr)
+	if second == -1 {
+		return -1
+	}
+	return first + second + 1
+}
+
 // splitPath splits a path into a top-level directory and a child item. The
 // parent directory retains the trailing slash.
-func splitPath(s string, lastIndex bool) (string, string) {
+func splitPath(s string, secondIndex bool) (string, string) {
 	var i int
-	if lastIndex {
-		i = strings.LastIndex(s, "/")
+	if secondIndex {
+		i = findSecondIndex(s, "/")
 	} else {
 		i = strings.Index(s, "/")
 	}
@@ -506,8 +518,8 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (res map
 			return nil, item.Err
 		}
 
-		lastIndex := strings.HasPrefix(item.Item, policyDBPrefix)
-		listKey, trimmedItem := splitPath(item.Item, lastIndex)
+		secondIndex := strings.HasPrefix(item.Item, policyDBPrefix)
+		listKey, trimmedItem := splitPath(item.Item, secondIndex)
 		if listKey == iamFormatFile {
 			continue
 		}
@@ -781,7 +793,10 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range listedConfigItems[stsListKey] {
 		userName := path.Dir(item)
 		// loadUser() will delete expired user during the load.
-		iamLogIf(ctx, iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore))
+		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
+		if err != nil && !errors.Is(err, errNoSuchUser) {
+			iamLogIf(ctx, err)
+		}
 		// No need to return errors for failed expiration of STS users
 	}
 
@@ -789,7 +804,10 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	// (removed during loadUser() in the loop above) are removed from memory.
 	for _, item := range listedConfigItems[policyDBSTSUsersListKey] {
 		stsName := strings.TrimSuffix(item, ".json")
-		iamLogIf(ctx, iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore))
+		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
+		if err != nil && !errors.Is(err, errNoSuchPolicy) {
+			iamLogIf(ctx, err)
+		}
 		// No need to return errors for failed expiration of STS users
 	}
 

cmd/iam-object-store_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+)
+
+func TestSplitPath(t *testing.T) {
+	cases := []struct {
+		path                          string
+		secondIndex                   bool
+		expectedListKey, expectedItem string
+	}{
+		{"format.json", false, "format.json", ""},
+		{"users/tester.json", false, "users/", "tester.json"},
+		{"groups/test/group.json", false, "groups/", "test/group.json"},
+		{"policydb/groups/testgroup.json", true, "policydb/groups/", "testgroup.json"},
+		{
+			"policydb/sts-users/uid=slash/user,ou=people,ou=swengg,dc=min,dc=io.json", true,
+			"policydb/sts-users/", "uid=slash/user,ou=people,ou=swengg,dc=min,dc=io.json",
+		},
+		{
+			"policydb/sts-users/uid=slash/user/twice,ou=people,ou=swengg,dc=min,dc=io.json", true,
+			"policydb/sts-users/", "uid=slash/user/twice,ou=people,ou=swengg,dc=min,dc=io.json",
+		},
+		{
+			"policydb/groups/cn=project/d,ou=groups,ou=swengg,dc=min,dc=io.json", true,
+			"policydb/groups/", "cn=project/d,ou=groups,ou=swengg,dc=min,dc=io.json",
+		},
+	}
+	for i, test := range cases {
+		listKey, item := splitPath(test.path, test.secondIndex)
+		if listKey != test.expectedListKey || item != test.expectedItem {
+			t.Errorf("unexpected result on test[%v]: expected[%s, %s] but got [%s, %s]", i, test.expectedListKey, test.expectedItem, listKey, item)
+		}
+	}
+}

cmd/iam-store.go

@@ -26,6 +26,7 @@ import (
 	"path"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -37,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/policy"
+	"github.com/minio/pkg/v3/sync/errgroup"
 	"github.com/puzpuzpuz/xsync/v3"
 	"golang.org/x/sync/singleflight"
 )
@@ -357,6 +359,68 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
 	}
 }
 
+func (c *iamCache) policyDBGetGroups(store *IAMStoreSys, userPolicyPresent bool, groups ...string) ([]string, error) {
+	var policies []string
+	for _, group := range groups {
+		if store.getUsersSysType() == MinIOUsersSysType {
+			g, ok := c.iamGroupsMap[group]
+			if !ok {
+				continue
+			}
+
+			// Group is disabled, so we return no policy - this
+			// ensures the request is denied.
+			if g.Status == statusDisabled {
+				continue
+			}
+		}
+
+		policy, ok := c.iamGroupPolicyMap.Load(group)
+		if !ok {
+			continue
+		}
+
+		policies = append(policies, policy.toSlice()...)
+	}
+
+	found := len(policies) > 0
+	if found {
+		return policies, nil
+	}
+
+	if userPolicyPresent {
+		// if user mapping present and no group policies found
+		// rely on user policy for access, instead of fallback.
+		return nil, nil
+	}
+
+	var mu sync.Mutex
+
+	// no mappings found, fallback for all groups.
+	g := errgroup.WithNErrs(len(groups)).WithConcurrency(10) // load like 10 groups at a time.
+
+	for index := range groups {
+		index := index
+		g.Go(func() error {
+			err := store.loadMappedPolicy(context.TODO(), groups[index], regUser, true, c.iamGroupPolicyMap)
+			if err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return err
+			}
+			if errors.Is(err, errNoSuchPolicy) {
+				return nil
+			}
+			policy, _ := c.iamGroupPolicyMap.Load(groups[index])
+			mu.Lock()
+			policies = append(policies, policy.toSlice()...)
+			mu.Unlock()
+			return nil
+		}, index)
+	}
+
+	err := errors.Join(g.Wait()...)
+	return policies, err
+}
+
 // policyDBGet - lower-level helper; does not take locks.
 //
 // If a group is passed, it returns policies associated with the group.
@@ -676,7 +740,8 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
 type IAMStoreSys struct {
 	IAMStorageAPI
 
-	group *singleflight.Group
+	group  *singleflight.Group
+	policy *singleflight.Group
 }
 
 // HasWatcher - returns if the storage system has a watcher.
@@ -757,21 +822,32 @@ func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string,
 	cache := store.rlock()
 	defer store.runlock()
 
-	policies, _, err := cache.policyDBGet(store, name, false, false)
-	if err != nil {
-		return nil, err
-	}
-
-	userPolicyPresent := len(policies) > 0
-	for _, group := range groups {
-		ps, _, err := cache.policyDBGet(store, group, true, userPolicyPresent)
+	getPolicies := func() ([]string, error) {
+		policies, _, err := cache.policyDBGet(store, name, false, false)
 		if err != nil {
 			return nil, err
 		}
-		policies = append(policies, ps...)
-	}
 
-	return policies, nil
+		userPolicyPresent := len(policies) > 0
+
+		groupPolicies, err := cache.policyDBGetGroups(store, userPolicyPresent, groups...)
+		if err != nil {
+			return nil, err
+		}
+
+		policies = append(policies, groupPolicies...)
+		return policies, nil
+	}
+	if store.policy != nil {
+		val, err, _ := store.policy.Do(name, func() (interface{}, error) {
+			return getPolicies()
+		})
+		if err != nil {
+			return nil, err
+		}
+		return val.([]string), nil
+	}
+	return getPolicies()
 }
 
 // AddUsersToGroup - adds users to group, creating the group if needed.
@@ -2009,7 +2085,8 @@ type ParentUserInfo struct {
 // GetAllParentUsers - returns all distinct "parent-users" associated with STS
 // or service credentials, mapped to all distinct roleARNs associated with the
 // parent user. The dummy role ARN is associated with parent users from
-// policy-claim based OpenID providers.
+// policy-claim based OpenID providers. The root credential as a parent
+// user is not included in the result.
 func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	cache := store.rlock()
 	defer store.runlock()
@@ -2031,7 +2108,7 @@ func (store *IAMStoreSys) getParentUsers(cache *iamCache) map[string]ParentUserI
 
 		var (
 			err    error
-			claims map[string]interface{} = cred.Claims
+			claims *jwt.MapClaims
 		)
 
 		if cred.IsServiceAccount() {
@@ -2048,29 +2125,22 @@ func (store *IAMStoreSys) getParentUsers(cache *iamCache) map[string]ParentUserI
 		if err != nil {
 			continue
 		}
-		if cred.ParentUser == "" {
+		if cred.ParentUser == "" || cred.ParentUser == globalActiveCred.AccessKey {
 			continue
 		}
 
 		subClaimValue := cred.ParentUser
-		if v, ok := claims[subClaim]; ok {
-			subFromToken, ok := v.(string)
-			if ok {
-				subClaimValue = subFromToken
-			}
+		if v, ok := claims.Lookup(subClaim); ok {
+			subClaimValue = v
 		}
-		if v, ok := claims[ldapActualUser]; ok {
-			subFromToken, ok := v.(string)
-			if ok {
-				subClaimValue = subFromToken
-			}
+		if v, ok := claims.Lookup(ldapActualUser); ok {
+			subClaimValue = v
 		}
 
 		roleArn := openid.DummyRoleARN.String()
-		s, ok := claims[roleArnClaim]
-		val, ok2 := s.(string)
-		if ok && ok2 {
-			roleArn = val
+		s, ok := claims.Lookup(roleArnClaim)
+		if ok {
+			roleArn = s
 		}
 		v, ok := res[cred.ParentUser]
 		if ok {
@@ -2537,13 +2607,15 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	// Extracted session policy name string can be removed as its not useful
 	// at this point.
-	delete(m, sessionPolicyNameExtracted)
+	m.Delete(sessionPolicyNameExtracted)
+
+	nosp := opts.sessionPolicy == nil || opts.sessionPolicy.Version == "" && len(opts.sessionPolicy.Statements) == 0
 
 	// sessionPolicy is nil and there is embedded policy attached we remove
 	// embedded policy at that point.
-	if _, ok := m[policy.SessionPolicyName]; ok && opts.sessionPolicy == nil {
-		delete(m, policy.SessionPolicyName)
-		m[iamPolicyClaimNameSA()] = inheritedPolicyType
+	if _, ok := m.Lookup(policy.SessionPolicyName); ok && nosp {
+		m.Delete(policy.SessionPolicyName)
+		m.Set(iamPolicyClaimNameSA(), inheritedPolicyType)
 	}
 
 	if opts.sessionPolicy != nil { // session policies is being updated
@@ -2551,21 +2623,23 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 			return updatedAt, err
 		}
 
-		policyBuf, err := json.Marshal(opts.sessionPolicy)
-		if err != nil {
-			return updatedAt, err
-		}
+		if opts.sessionPolicy.Version != "" && len(opts.sessionPolicy.Statements) > 0 {
+			policyBuf, err := json.Marshal(opts.sessionPolicy)
+			if err != nil {
+				return updatedAt, err
+			}
 
-		if len(policyBuf) > maxSVCSessionPolicySize {
-			return updatedAt, errSessionPolicyTooLarge
-		}
+			if len(policyBuf) > maxSVCSessionPolicySize {
+				return updatedAt, errSessionPolicyTooLarge
+			}
 
-		// Overwrite session policy claims.
-		m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
-		m[iamPolicyClaimNameSA()] = embeddedPolicyType
+			// Overwrite session policy claims.
+			m.Set(policy.SessionPolicyName, base64.StdEncoding.EncodeToString(policyBuf))
+			m.Set(iamPolicyClaimNameSA(), embeddedPolicyType)
+		}
 	}
 
-	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
+	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m.Map(), cr.SecretKey)
 	if err != nil {
 		return updatedAt, err
 	}
@@ -2892,22 +2966,22 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error
 
 func extractJWTClaims(u UserIdentity) (jwtClaims *jwt.MapClaims, err error) {
 	keys := make([]string, 0, 3)
+
 	// Append credentials secret key itself
 	keys = append(keys, u.Credentials.SecretKey)
+
 	// Use site-replication credentials if found
 	if globalSiteReplicationSys.isEnabled() {
-		siteReplSecretKey, e := globalSiteReplicatorCred.Get(GlobalContext)
-		if e != nil {
-			return nil, e
+		secretKey, err := getTokenSigningKey()
+		if err != nil {
+			return nil, err
 		}
-		keys = append(keys, siteReplSecretKey)
+		keys = append(keys, secretKey)
 	}
-	// Use root credentials for credentials created with older deployments
-	keys = append(keys, globalActiveCred.SecretKey)
 
 	// Iterate over all keys and return with the first successful claim extraction
 	for _, key := range keys {
-		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, key)
+		jwtClaims, err = getClaimsFromTokenWithSecret(u.Credentials.SessionToken, key)
 		if err == nil {
 			break
 		}

cmd/iam.go

@@ -178,13 +178,18 @@ func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
 	}
 
 	if etcdClient == nil {
-		var group *singleflight.Group
+		var (
+			group  *singleflight.Group
+			policy *singleflight.Group
+		)
 		if env.Get("_MINIO_IAM_SINGLE_FLIGHT", config.EnableOn) == config.EnableOn {
 			group = &singleflight.Group{}
+			policy = &singleflight.Group{}
 		}
 		sys.store = &IAMStoreSys{
 			IAMStorageAPI: newIAMObjectStore(objAPI, sys.usersSysType),
 			group:         group,
+			policy:        policy,
 		}
 	} else {
 		sys.store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(etcdClient, sys.usersSysType)}
@@ -406,6 +411,7 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 	timer := time.NewTimer(waitInterval())
 	defer timer.Stop()
 
+	lastPurgeHour := -1
 	for {
 		select {
 		case <-timer.C:
@@ -421,9 +427,9 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 				}
 			}
 
-			// The following actions are performed about once in 4 times that
-			// IAM is refreshed:
-			if r.Intn(4) == 0 {
+			// Run purge routines once in each hour.
+			if refreshStart.Hour() != lastPurgeHour {
+				lastPurgeHour = refreshStart.Hour()
 				// Poll and remove accounts for those users who were removed
 				// from LDAP/OpenID.
 				if sys.LDAPConfig.Enabled() {
@@ -1294,10 +1300,6 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errServerNotInitialized
 	}
 
-	if sys.usersSysType != LDAPUsersSysType {
-		return nil, nil
-	}
-
 	sa, ok := sys.store.GetUser(accessKey)
 	if !ok || !sa.Credentials.IsServiceAccount() {
 		return nil, errNoSuchServiceAccount
@@ -2179,7 +2181,6 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 			return false
 		}
 		svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
-
 	default:
 		// Check policy for parent user of service account.
 		svcPolicies, err = sys.PolicyDBGet(parentUser, args.Groups...)

cmd/metacache-set.go

@@ -29,6 +29,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -182,8 +183,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 	resultsDone := make(chan metaCacheEntriesSorted)
 	// Copy so we can mutate
 	resCh := resultsDone
-	var done bool
-	var mu sync.Mutex
+	var done atomic.Bool
 	resErr := io.EOF
 
 	go func() {
@@ -194,9 +194,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 				// past limit
 				continue
 			}
-			mu.Lock()
-			returned = done
-			mu.Unlock()
+			returned = done.Load()
 			if returned {
 				resCh = nil
 				continue
@@ -250,9 +248,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 	return func() (metaCacheEntriesSorted, error) {
 		select {
 		case <-ctx.Done():
-			mu.Lock()
-			done = true
-			mu.Unlock()
+			done.Store(true)
 			return metaCacheEntriesSorted{}, ctx.Err()
 		case r := <-resultsDone:
 			return r, resErr
@@ -626,18 +622,18 @@ func calcCommonWritesDeletes(infos []DiskInfo, readQuorum int) (commonWrite, com
 	}
 
 	filter := func(list []uint64) (commonCount uint64) {
-		max := 0
+		maxCnt := 0
 		signatureMap := map[uint64]int{}
 		for _, v := range list {
 			signatureMap[v]++
 		}
 		for ops, count := range signatureMap {
-			if max < count && commonCount < ops {
-				max = count
+			if maxCnt < count && commonCount < ops {
+				maxCnt = count
 				commonCount = ops
 			}
 		}
-		if max < readQuorum {
+		if maxCnt < readQuorum {
 			return 0
 		}
 		return commonCount
@@ -650,7 +646,7 @@ func calcCommonWritesDeletes(infos []DiskInfo, readQuorum int) (commonWrite, com
 
 func calcCommonCounter(infos []DiskInfo, readQuorum int) (commonCount uint64) {
 	filter := func() (commonCount uint64) {
-		max := 0
+		maxCnt := 0
 		signatureMap := map[uint64]int{}
 		for _, info := range infos {
 			if info.Error != "" {
@@ -660,12 +656,12 @@ func calcCommonCounter(infos []DiskInfo, readQuorum int) (commonCount uint64) {
 			signatureMap[mutations]++
 		}
 		for ops, count := range signatureMap {
-			if max < count && commonCount < ops {
-				max = count
+			if maxCnt < count && commonCount < ops {
+				maxCnt = count
 				commonCount = ops
 			}
 		}
-		if max < readQuorum {
+		if maxCnt < readQuorum {
 			return 0
 		}
 		return commonCount

cmd/metrics-v2.go

@@ -1875,13 +1875,17 @@ func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription,
 		// add metrics with +Inf label
 		labels1 := make(map[string]string)
 		for _, lp := range dtoMetric.GetLabel() {
-			labels1[*lp.Name] = *lp.Value
+			if *lp.Name == "api" && toLowerAPILabels {
+				labels1[*lp.Name] = strings.ToLower(*lp.Value)
+			} else {
+				labels1[*lp.Name] = *lp.Value
+			}
 		}
 		labels1["le"] = fmt.Sprintf("%.3f", math.Inf(+1))
 		metrics = append(metrics, MetricV2{
 			Description:    desc,
 			VariableLabels: labels1,
-			Value:          dtoMetric.Counter.GetValue(),
+			Value:          float64(dtoMetric.Histogram.GetSampleCount()),
 		})
 	}
 	return metrics

cmd/metrics-v2_test.go

@@ -18,13 +18,15 @@
 package cmd
 
 import (
+	"slices"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-func TestGetHistogramMetrics(t *testing.T) {
+func TestGetHistogramMetrics_BucketCount(t *testing.T) {
 	histBuckets := []float64{0.05, 0.1, 0.25, 0.5, 0.75}
 	labels := []string{"GetObject", "PutObject", "CopyObject", "CompleteMultipartUpload"}
 	ttfbHist := prometheus.NewHistogramVec(
@@ -83,6 +85,135 @@ func TestGetHistogramMetrics(t *testing.T) {
 	metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false)
 	// additional labels for +Inf for all histogram metrics
 	if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) {
-		t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics))
+		t.Fatalf("Expected %v data points when toLowerAPILabels=false but got %v", expPoints, len(metrics))
+	}
+
+	metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true)
+	// additional labels for +Inf for all histogram metrics
+	if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) {
+		t.Fatalf("Expected %v data points when toLowerAPILabels=true but got %v", expPoints, len(metrics))
+	}
+}
+
+func TestGetHistogramMetrics_Values(t *testing.T) {
+	histBuckets := []float64{0.50, 5.00}
+	labels := []string{"PutObject", "CopyObject"}
+	ttfbHist := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "s3_ttfb_seconds",
+			Help:    "Time taken by requests served by current MinIO server instance",
+			Buckets: histBuckets,
+		},
+		[]string{"api"},
+	)
+	observations := []struct {
+		val   float64
+		label string
+	}{
+		{
+			val:   0.02,
+			label: labels[0],
+		},
+		{
+			val:   0.19,
+			label: labels[1],
+		},
+		{
+			val:   0.31,
+			label: labels[1],
+		},
+		{
+			val:   0.61,
+			label: labels[0],
+		},
+		{
+			val:   6.79,
+			label: labels[1],
+		},
+	}
+	ticker := time.NewTicker(1 * time.Millisecond)
+	defer ticker.Stop()
+	for _, obs := range observations {
+		// Send observations once every 1ms, to simulate delay between
+		// observations. This is to test the channel based
+		// synchronization used internally.
+		select {
+		case <-ticker.C:
+			ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val)
+		}
+	}
+
+	// Accumulate regular-cased API label metrics for 'PutObject' for deeper verification
+	metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false)
+	capitalPutObjects := make([]MetricV2, 0, len(histBuckets)+1)
+	for _, metric := range metrics {
+		if value := metric.VariableLabels["api"]; value == "PutObject" {
+			capitalPutObjects = append(capitalPutObjects, metric)
+		}
+	}
+	if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(capitalPutObjects) {
+		t.Fatalf("Expected %d api=PutObject metrics but got %d", expMetricsPerAPI, len(capitalPutObjects))
+	}
+
+	// Deterministic ordering
+	slices.SortFunc(capitalPutObjects, func(a MetricV2, b MetricV2) int {
+		le1 := a.VariableLabels["le"]
+		le2 := a.VariableLabels["le"]
+		return strings.Compare(le1, le2)
+	})
+	if le := capitalPutObjects[0].VariableLabels["le"]; le != "0.500" {
+		t.Errorf("Expected le='0.050' api=PutObject metrics but got '%v'", le)
+	}
+	if value := capitalPutObjects[0].Value; value != 1 {
+		t.Errorf("Expected le='0.050' api=PutObject value to be 1 but got '%v'", value)
+	}
+	if le := capitalPutObjects[1].VariableLabels["le"]; le != "5.000" {
+		t.Errorf("Expected le='5.000' api=PutObject metrics but got '%v'", le)
+	}
+	if value := capitalPutObjects[1].Value; value != 2 {
+		t.Errorf("Expected le='5.000' api=PutObject value to be 2 but got '%v'", value)
+	}
+	if le := capitalPutObjects[2].VariableLabels["le"]; le != "+Inf" {
+		t.Errorf("Expected le='+Inf' api=PutObject metrics but got '%v'", le)
+	}
+	if value := capitalPutObjects[2].Value; value != 2 {
+		t.Errorf("Expected le='+Inf' api=PutObject value to be 2 but got '%v'", value)
+	}
+
+	// Accumulate lower-cased API label metrics for 'copyobject' for deeper verification
+	metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true)
+	lowerCopyObjects := make([]MetricV2, 0, len(histBuckets)+1)
+	for _, metric := range metrics {
+		if value := metric.VariableLabels["api"]; value == "copyobject" {
+			lowerCopyObjects = append(lowerCopyObjects, metric)
+		}
+	}
+	if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(lowerCopyObjects) {
+		t.Fatalf("Expected %d api=copyobject metrics but got %d", expMetricsPerAPI, len(lowerCopyObjects))
+	}
+
+	// Deterministic ordering
+	slices.SortFunc(lowerCopyObjects, func(a MetricV2, b MetricV2) int {
+		le1 := a.VariableLabels["le"]
+		le2 := a.VariableLabels["le"]
+		return strings.Compare(le1, le2)
+	})
+	if le := lowerCopyObjects[0].VariableLabels["le"]; le != "0.500" {
+		t.Errorf("Expected le='0.050' api=copyobject metrics but got '%v'", le)
+	}
+	if value := lowerCopyObjects[0].Value; value != 2 {
+		t.Errorf("Expected le='0.050' api=copyobject value to be 2 but got '%v'", value)
+	}
+	if le := lowerCopyObjects[1].VariableLabels["le"]; le != "5.000" {
+		t.Errorf("Expected le='5.000' api=copyobject metrics but got '%v'", le)
+	}
+	if value := lowerCopyObjects[1].Value; value != 2 {
+		t.Errorf("Expected le='5.000' api=copyobject value to be 2 but got '%v'", value)
+	}
+	if le := lowerCopyObjects[2].VariableLabels["le"]; le != "+Inf" {
+		t.Errorf("Expected le='+Inf' api=copyobject metrics but got '%v'", le)
+	}
+	if value := lowerCopyObjects[2].Value; value != 3 {
+		t.Errorf("Expected le='+Inf' api=copyobject value to be 3 but got '%v'", value)
 	}
 }

cmd/metrics-v3-api.go

@@ -129,7 +129,7 @@ func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsC
 // This is a `MetricsLoaderFn`.
 func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
 	renameLabels := map[string]string{"api": "name"}
-	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	labelsFilter := map[string]set.StringSet{}
 	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil,
 		"type", "s3")
 	return nil
@@ -217,7 +217,7 @@ func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCac
 // This is a `MetricsLoaderFn`.
 func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
 	renameLabels := map[string]string{"api": "name"}
-	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	labelsFilter := map[string]set.StringSet{}
 	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels,
 		buckets, "type", "s3")
 	return nil

cmd/metrics-v3-types.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"slices"
 	"strings"
 	"sync"
 
@@ -27,7 +28,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
-	"golang.org/x/exp/slices"
 )
 
 type collectorPath string

cmd/namespace-lock.go

@@ -229,6 +229,7 @@ type localLockInstance struct {
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
 func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume string, paths ...string) RWLocker {
+	sort.Strings(paths)
 	opsID := mustGetUUID()
 	if n.isDistErasure {
 		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
@@ -237,7 +238,6 @@ func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume
 		}, pathsJoinPrefix(volume, paths...)...)
 		return &distLockInstance{drwmutex, opsID}
 	}
-	sort.Strings(paths)
 	return &localLockInstance{n, volume, paths, opsID}
 }
 

cmd/object-api-options.go

@@ -404,12 +404,7 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
 		metadata = make(map[string]string)
 	}
 
-	wantCRC, err := hash.GetContentChecksum(hdr)
-	if err != nil {
-		return opts, fmt.Errorf("invalid/unknown checksum sent: %v", err)
-	}
 	etag := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceETag))
-
 	if crypto.S3KMS.IsRequested(hdr) {
 		keyID, context, err := crypto.S3KMS.ParseHTTP(hdr)
 		if err != nil {
@@ -423,7 +418,6 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
 			ServerSideEncryption: sseKms,
 			UserDefined:          metadata,
 			MTime:                mtime,
-			WantChecksum:         wantCRC,
 			PreserveETag:         etag,
 		}, nil
 	}
@@ -438,7 +432,6 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
 	opts.ReplicationSourceRetentionTimestamp = retaintimestmp
 	opts.ReplicationSourceTaggingTimestamp = taggingtimestmp
 	opts.PreserveETag = etag
-	opts.WantChecksum = wantCRC
 
 	return opts, nil
 }

cmd/object-api-utils.go

@@ -29,6 +29,7 @@ import (
 	"net/http"
 	"path"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -52,7 +53,6 @@ import (
 	"github.com/minio/pkg/v3/trie"
 	"github.com/minio/pkg/v3/wildcard"
 	"github.com/valyala/bytebufferpool"
-	"golang.org/x/exp/slices"
 )
 
 const (

cmd/object-handlers-common.go

@@ -352,11 +352,11 @@ func isETagEqual(left, right string) bool {
 // setPutObjHeaders sets all the necessary headers returned back
 // upon a success Put/Copy/CompleteMultipart/Delete requests
 // to activate delete only headers set delete as true
-func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool, h http.Header) {
+func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, del bool, h http.Header) {
 	// We must not use the http.Header().Set method here because some (broken)
 	// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
 	// Therefore, we have to set the ETag directly as map entry.
-	if objInfo.ETag != "" && !delete {
+	if objInfo.ETag != "" && !del {
 		w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
 	}
 
@@ -364,13 +364,13 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool, h
 	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 		// If version is a deleted marker, set this header as well
-		if objInfo.DeleteMarker && delete { // only returned during delete object
+		if objInfo.DeleteMarker && del { // only returned during delete object
 			w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)}
 		}
 	}
 
 	if objInfo.Bucket != "" && objInfo.Name != "" {
-		if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !delete {
+		if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !del {
 			lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 		}
 	}

cmd/object-handlers.go

@@ -1191,6 +1191,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	// Sanitize the source object name similar to NewMultipart and PutObject API
+	srcObject = trimLeadingSlash(srcObject)
+
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
@@ -1522,7 +1525,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			if !srcTimestamp.IsZero() {
 				ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastTaggingTimestamp)
 				// update tagging metadata only if replica  timestamp is newer than what's on disk
-				if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) {
+				if err != nil || (err == nil && !ondiskTimestamp.After(srcTimestamp)) {
 					srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
 					srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
 				}
@@ -1899,6 +1902,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	var reader io.Reader
 	reader = rd
 
+	var opts ObjectOptions
+	opts, err = putOptsFromReq(ctx, r, bucket, object, metadata)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	actualSize := size
 	var idxCb func() []byte
 	if isCompressible(r.Header, object) && size > minCompressibleSize {
@@ -1915,6 +1925,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
 			return
 		}
+		opts.WantChecksum = actualReader.Checksum()
+
 		// Set compression metrics.
 		var s2c io.ReadCloser
 		wantEncryption := crypto.Requested(r.Header)
@@ -1945,20 +1957,16 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err := hashReader.AddChecksum(r, size < 0); err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
-		return
+	if size >= 0 {
+		if err := hashReader.AddChecksum(r, false); err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
+			return
+		}
+		opts.WantChecksum = hashReader.Checksum()
 	}
 
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader)
-
-	var opts ObjectOptions
-	opts, err = putOptsFromReq(ctx, r, bucket, object, metadata)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
 	opts.IndexCB = idxCb
 
 	if opts.PreserveETag != "" ||
@@ -2898,6 +2906,8 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
+	reqInfo := logger.GetReqInfo(ctx)
+	reqInfo.SetTags("retention", objRetention.String())
 
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {

cmd/os_windows.go

@@ -60,20 +60,19 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 		if err != nil {
 			if err == syscall.ERROR_NO_MORE_FILES {
 				break
-			} else {
-				if isSysErrPathNotFound(err) {
-					return nil
-				}
-				err = osErrToFileErr(&os.PathError{
-					Op:   "FindNextFile",
-					Path: dirPath,
-					Err:  err,
-				})
-				if err == errFileNotFound {
-					return nil
-				}
-				return err
 			}
+			if isSysErrPathNotFound(err) {
+				return nil
+			}
+			err = osErrToFileErr(&os.PathError{
+				Op:   "FindNextFile",
+				Path: dirPath,
+				Err:  err,
+			})
+			if err == errFileNotFound {
+				return nil
+			}
+			return err
 		}
 		name := syscall.UTF16ToString(data.FileName[0:])
 		if name == "" || name == "." || name == ".." { // Useless names
@@ -136,13 +135,12 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 		if err != nil {
 			if err == syscall.ERROR_NO_MORE_FILES {
 				break
-			} else {
-				return nil, osErrToFileErr(&os.PathError{
-					Op:   "FindNextFile",
-					Path: dirPath,
-					Err:  err,
-				})
 			}
+			return nil, osErrToFileErr(&os.PathError{
+				Op:   "FindNextFile",
+				Path: dirPath,
+				Err:  err,
+			})
 		}
 
 		name := syscall.UTF16ToString(data.FileName[0:])

cmd/peer-rest-server.go

@@ -56,6 +56,7 @@ var (
 	aoBucketInfo           = grid.NewArrayOf[*BucketInfo](func() *BucketInfo { return &BucketInfo{} })
 	aoMetricsGroup         = grid.NewArrayOf[*MetricV2](func() *MetricV2 { return &MetricV2{} })
 	madminBgHealState      = grid.NewJSONPool[madmin.BgHealState]()
+	madminHealResultItem   = grid.NewJSONPool[madmin.HealResultItem]()
 	madminCPUs             = grid.NewJSONPool[madmin.CPUs]()
 	madminMemInfo          = grid.NewJSONPool[madmin.MemInfo]()
 	madminNetInfo          = grid.NewJSONPool[madmin.NetInfo]()
@@ -97,7 +98,7 @@ var (
 	getSysErrorsRPC                = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.SysErrors]](grid.HandlerGetSysErrors, grid.NewMSS, madminSysErrors.NewJSON)
 	getSysServicesRPC              = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.SysServices]](grid.HandlerGetSysServices, grid.NewMSS, madminSysServices.NewJSON)
 	headBucketRPC                  = grid.NewSingleHandler[*grid.MSS, *VolInfo](grid.HandlerHeadBucket, grid.NewMSS, func() *VolInfo { return &VolInfo{} })
-	healBucketRPC                  = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerHealBucket, grid.NewMSS, grid.NewNoPayload)
+	healBucketRPC                  = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.HealResultItem]](grid.HandlerHealBucket, grid.NewMSS, madminHealResultItem.NewJSON)
 	listBucketsRPC                 = grid.NewSingleHandler[*BucketOptions, *grid.Array[*BucketInfo]](grid.HandlerListBuckets, func() *BucketOptions { return &BucketOptions{} }, aoBucketInfo.New)
 	loadBucketMetadataRPC          = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadBucketMetadata, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn()
 	loadGroupRPC                   = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadGroup, grid.NewMSS, grid.NewNoPayload)
@@ -636,7 +637,7 @@ func (s *peerRESTServer) VerifyBinaryHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	if lrTime.Sub(currentReleaseTime) <= 0 {
-		s.writeErrorResponse(w, fmt.Errorf("server is already running the latest version: %s", Version))
+		s.writeErrorResponse(w, fmt.Errorf("server is running the latest version: %s", Version))
 		return
 	}
 
@@ -1258,21 +1259,21 @@ func (s *peerRESTServer) NetSpeedTestHandler(w http.ResponseWriter, r *http.Requ
 	peersLogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }
 
-func (s *peerRESTServer) HealBucketHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) {
+func (s *peerRESTServer) HealBucketHandler(mss *grid.MSS) (np *grid.JSON[madmin.HealResultItem], nerr *grid.RemoteErr) {
 	bucket := mss.Get(peerS3Bucket)
 	if isMinioMetaBucket(bucket) {
 		return np, grid.NewRemoteErr(errInvalidArgument)
 	}
 
 	bucketDeleted := mss.Get(peerS3BucketDeleted) == "true"
-	_, err := healBucketLocal(context.Background(), bucket, madmin.HealOpts{
+	res, err := healBucketLocal(context.Background(), bucket, madmin.HealOpts{
 		Remove: bucketDeleted,
 	})
 	if err != nil {
 		return np, grid.NewRemoteErr(err)
 	}
 
-	return np, nil
+	return madminHealResultItem.NewJSONWith(&res), nil
 }
 
 func (s *peerRESTServer) ListBucketsHandler(opts *BucketOptions) (*grid.Array[*BucketInfo], *grid.RemoteErr) {

cmd/peer-s3-client.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"sort"
 	"strconv"
 	"sync/atomic"
@@ -29,7 +30,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/grid"
 	"github.com/minio/pkg/v3/sync/errgroup"
-	"golang.org/x/exp/slices"
 )
 
 var errPeerOffline = errors.New("peer is offline")
@@ -178,13 +178,24 @@ func (sys *S3PeerSys) HealBucket(ctx context.Context, bucket string, opts madmin
 		}
 	}
 
+	if healBucketErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, len(errs)/2+1); healBucketErr != nil {
+		return madmin.HealResultItem{}, toObjectErr(healBucketErr, bucket)
+	}
+
+	res := madmin.HealResultItem{
+		Type:     madmin.HealItemBucket,
+		Bucket:   bucket,
+		SetCount: -1, // explicitly set an invalid value -1, for bucket heal scenario
+	}
+
 	for i, err := range errs {
 		if err == nil {
-			return healBucketResults[i], nil
+			res.Before.Drives = append(res.Before.Drives, healBucketResults[i].Before.Drives...)
+			res.After.Drives = append(res.After.Drives, healBucketResults[i].After.Drives...)
 		}
 	}
 
-	return madmin.HealResultItem{}, toObjectErr(errVolumeNotFound, bucket)
+	return res, nil
 }
 
 // ListBuckets lists buckets across all nodes and returns a consistent view:
@@ -355,14 +366,8 @@ func (client *remotePeerS3Client) HealBucket(ctx context.Context, bucket string,
 	ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
 	defer cancel()
 
-	_, err := healBucketRPC.Call(ctx, conn, mss)
-
-	// Initialize heal result info
-	return madmin.HealResultItem{
-		Type:     madmin.HealItemBucket,
-		Bucket:   bucket,
-		SetCount: -1, // explicitly set an invalid value -1, for bucket heal scenario
-	}, toStorageErr(err)
+	resp, err := healBucketRPC.Call(ctx, conn, mss)
+	return resp.ValueOrZero(), toStorageErr(err)
 }
 
 // GetBucketInfo returns bucket stat info from a peer

cmd/peer-s3-server.go

@@ -101,7 +101,7 @@ func healBucketLocal(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	for i := range beforeState {
 		res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: localDrives[i].String(),
+			Endpoint: localDrives[i].Endpoint().String(),
 			State:    beforeState[i],
 		})
 	}
@@ -149,7 +149,7 @@ func healBucketLocal(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	for i := range afterState {
 		res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: localDrives[i].String(),
+			Endpoint: localDrives[i].Endpoint().String(),
 			State:    afterState[i],
 		})
 	}

cmd/post-policy_test.go

@@ -56,10 +56,12 @@ func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey stri
 	credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
 	// Add the meta-uuid string, set to 1234
 	uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234")
+	// Add the content-encoding string, set to gzip.
+	contentEncodingConditionStr := fmt.Sprintf(`["eq", "$content-encoding", "%s"]`, "gzip")
 
 	// Combine all conditions into one string.
-	conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr,
-		keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr)
+	conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr,
+		keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr, contentEncodingConditionStr)
 	retStr := "{"
 	retStr = retStr + expirationStr + ","
 	retStr += conditionStr
@@ -85,9 +87,11 @@ func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration t
 	credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential)
 	// Add the meta-uuid string, set to 1234
 	uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234")
+	// Add the content-encoding string, set to gzip
+	contentEncodingConditionStr := fmt.Sprintf(`["eq", "$content-encoding", "%s"]`, "gzip")
 
 	// Combine all conditions into one string.
-	conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr)
+	conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr, contentEncodingConditionStr)
 	retStr := "{"
 	retStr = retStr + expirationStr + ","
 	retStr += conditionStr
@@ -331,7 +335,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			dates:              []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
-			policy:             `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`,
+			policy:             `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`,
 		},
 		// Success case, no multipart filename.
 		{
@@ -341,7 +345,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			dates:              []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
-			policy:             `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`,
+			policy:             `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`,
 			noFilename:         true,
 		},
 		// Success case, big body.
@@ -352,7 +356,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			dates:              []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},
-			policy:             `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"]]}`,
+			policy:             `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`,
 		},
 		// Corrupted Base 64 result
 		{
@@ -447,7 +451,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 			malformedBody:       false,
 			ignoreContentLength: false,
 		},
-		// Failed with Content-Length not specified.
+		// Success with Content-Length not specified.
 		{
 			objectName:          "test",
 			data:                bytes.Repeat([]byte("a"), 1025),
@@ -547,7 +551,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 	rec := httptest.NewRecorder()
 
 	dates := []interface{}{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}
-	policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`
+	policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$content-encoding", "gzip"]]}`
 
 	// Generate the final policy document
 	policy = fmt.Sprintf(policy, dates...)

cmd/postpolicyform.go

@@ -29,10 +29,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/bcicen/jstream"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/s3select/jstream"
 )
 
 // startWithConds - map which indicates if a given condition supports starts-with policy operator
@@ -53,18 +53,6 @@ var startsWithConds = map[string]bool{
 	"$x-amz-date":              false,
 }
 
-var postPolicyIgnoreKeys = map[string]bool{
-	"Policy":              true,
-	xhttp.AmzSignature:    true,
-	xhttp.ContentEncoding: true,
-	http.CanonicalHeaderKey(xhttp.AmzChecksumAlgo):   true,
-	http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32):  true,
-	http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32C): true,
-	http.CanonicalHeaderKey(xhttp.AmzChecksumSHA1):   true,
-	http.CanonicalHeaderKey(xhttp.AmzChecksumSHA256): true,
-	http.CanonicalHeaderKey(xhttp.AmzChecksumMode):   true,
-}
-
 // Add policy conditionals.
 const (
 	policyCondEqual         = "eq"
@@ -140,7 +128,7 @@ type PostPolicyForm struct {
 func sanitizePolicy(r io.Reader) (io.Reader, error) {
 	var buf bytes.Buffer
 	e := json.NewEncoder(&buf)
-	d := jstream.NewDecoder(r, 0).ObjectAsKVS()
+	d := jstream.NewDecoder(r, 0).ObjectAsKVS().MaxDepth(10)
 	sset := set.NewStringSet()
 	for mv := range d.Stream() {
 		var kvs jstream.KVS
@@ -232,19 +220,19 @@ func parsePostPolicyForm(r io.Reader) (PostPolicyForm, error) {
 					operator, matchType, value,
 				})
 			case policyCondContentLength:
-				min, err := toInteger(condt[1])
+				minLen, err := toInteger(condt[1])
 				if err != nil {
 					return parsedPolicy, err
 				}
 
-				max, err := toInteger(condt[2])
+				maxLen, err := toInteger(condt[2])
 				if err != nil {
 					return parsedPolicy, err
 				}
 
 				parsedPolicy.Conditions.ContentLengthRange = contentLengthRange{
-					Min:   min,
-					Max:   max,
+					Min:   minLen,
+					Max:   maxLen,
 					Valid: true,
 				}
 			default:
@@ -272,60 +260,57 @@ func checkPolicyCond(op string, input1, input2 string) bool {
 	return false
 }
 
+// S3 docs: "Each form field that you specify in a form (except x-amz-signature, file, policy, and field names
+// that have an x-ignore- prefix) must appear in the list of conditions."
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
+// keyInPolicyExceptions - list of keys that, when present in the form, can be missing in the conditions of the policy.
+var keyInPolicyExceptions = map[string]bool{
+	xhttp.AmzSignature: true,
+	"File":             true,
+	"Policy":           true,
+
+	// MinIO specific exceptions to the general S3 rule above.
+	encrypt.SseKmsKeyID:          true,
+	encrypt.SseEncryptionContext: true,
+	encrypt.SseCustomerAlgorithm: true,
+	encrypt.SseCustomerKey:       true,
+	encrypt.SseCustomerKeyMD5:    true,
+}
+
 // checkPostPolicy - apply policy conditions and validate input values.
-// (http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html)
+// Note that content-length-range is checked in the API handler function PostPolicyBucketHandler.
+// formValues is the already-canonicalized form values from the POST request.
 func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) error {
 	// Check if policy document expiry date is still not reached
 	if !postPolicyForm.Expiration.After(UTCNow()) {
 		return fmt.Errorf("Invalid according to Policy: Policy expired")
 	}
-	// check all formValues appear in postPolicyForm or return error. #https://github.com/minio/minio/issues/17391
-	checkHeader := map[string][]string{}
-	ignoreKeys := map[string]bool{}
-	for key, value := range formValues {
-		switch {
-		case ignoreKeys[key], postPolicyIgnoreKeys[key], strings.HasPrefix(key, encrypt.SseGenericHeader):
-			continue
-		case strings.HasPrefix(key, "X-Amz-Ignore-"):
-			ignoreKey := strings.Replace(key, "X-Amz-Ignore-", "", 1)
-			ignoreKeys[ignoreKey] = true
-			// if it have already
-			delete(checkHeader, ignoreKey)
-		default:
-			checkHeader[key] = value
-		}
-	}
-	// map to store the metadata
-	metaMap := make(map[string]string)
-	for _, policy := range postPolicyForm.Conditions.Policies {
-		if strings.HasPrefix(policy.Key, "$x-amz-meta-") {
-			formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$"))
-			metaMap[formCanonicalName] = policy.Value
-		}
-	}
-	// Check if any extra metadata field is passed as input
-	for key := range formValues {
-		if strings.HasPrefix(key, "X-Amz-Meta-") {
-			if _, ok := metaMap[key]; !ok {
-				return fmt.Errorf("Invalid according to Policy: Extra input fields: %s", key)
-			}
-		}
-	}
 
-	// Flag to indicate if all policies conditions are satisfied
-	var condPassed bool
+	// mustFindInPolicy is a map to list all the keys that we must find in the policy as
+	// we process it below. At the end of checkPostPolicy function, if any key is left in
+	// this map, that's an error.
+	mustFindInPolicy := make(map[string][]string, len(formValues))
+	for key, values := range formValues {
+		if keyInPolicyExceptions[key] || strings.HasPrefix(key, "X-Ignore-") {
+			continue
+		}
+		mustFindInPolicy[key] = values
+	}
 
 	// Iterate over policy conditions and check them against received form fields
 	for _, policy := range postPolicyForm.Conditions.Policies {
 		// Form fields names are in canonical format, convert conditions names
 		// to canonical for simplification purpose, so `$key` will become `Key`
 		formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$"))
+
 		// Operator for the current policy condition
 		op := policy.Operator
-		// Multiple values should not occur
-		if len(checkHeader[formCanonicalName]) >= 2 {
-			return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]. FormValues have multiple values: [%s]", op, policy.Key, policy.Value, strings.Join(checkHeader[formCanonicalName], ", "))
+
+		// Multiple values are not allowed for a single form field
+		if len(mustFindInPolicy[formCanonicalName]) >= 2 {
+			return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]. FormValues have multiple values: [%s]", op, policy.Key, policy.Value, strings.Join(mustFindInPolicy[formCanonicalName], ", "))
 		}
+
 		// If the current policy condition is known
 		if startsWithSupported, condFound := startsWithConds[policy.Key]; condFound {
 			// Check if the current condition supports starts-with operator
@@ -333,35 +318,35 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}
 			// Check if current policy condition is satisfied
-			condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value)
-			if !condPassed {
+			if !checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) {
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}
 		} else if strings.HasPrefix(policy.Key, "$x-amz-meta-") || strings.HasPrefix(policy.Key, "$x-amz-") {
 			// This covers all conditions X-Amz-Meta-* and X-Amz-*
 			// Check if policy condition is satisfied
-			condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value)
-			if !condPassed {
+			if !checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) {
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, policy.Key, policy.Value)
 			}
 		}
-		delete(checkHeader, formCanonicalName)
+		delete(mustFindInPolicy, formCanonicalName)
 	}
-	// For SignV2 - Signature/AWSAccessKeyId field will be ignored.
+
+	// For SignV2 - Signature/AWSAccessKeyId fields do not need to be in the policy
 	if _, ok := formValues[xhttp.AmzSignatureV2]; ok {
-		delete(checkHeader, xhttp.AmzSignatureV2)
-		for k := range checkHeader {
+		delete(mustFindInPolicy, xhttp.AmzSignatureV2)
+		for k := range mustFindInPolicy {
 			// case-insensitivity for AWSAccessKeyId
 			if strings.EqualFold(k, xhttp.AmzAccessKeyID) {
-				delete(checkHeader, k)
+				delete(mustFindInPolicy, k)
 				break
 			}
 		}
 	}
 
-	if len(checkHeader) != 0 {
-		logKeys := make([]string, 0, len(checkHeader))
-		for key := range checkHeader {
+	// Check mustFindInPolicy to see if any key is left, if so, it was not found in policy and we return an error.
+	if len(mustFindInPolicy) != 0 {
+		logKeys := make([]string, 0, len(mustFindInPolicy))
+		for key := range mustFindInPolicy {
 			logKeys = append(logKeys, key)
 		}
 		return fmt.Errorf("Each form field that you specify in a form must appear in the list of policy conditions. %q not specified in the policy.", strings.Join(logKeys, ", "))

cmd/postpolicyform_test.go

@@ -20,12 +20,12 @@ package cmd
 import (
 	"bytes"
 	"encoding/base64"
-	"fmt"
 	"net/http"
 	"strings"
 	"testing"
 
 	minio "github.com/minio/minio-go/v7"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 func TestParsePostPolicyForm(t *testing.T) {
@@ -78,6 +78,28 @@ func TestParsePostPolicyForm(t *testing.T) {
 	}
 }
 
+type formValues struct {
+	http.Header
+}
+
+func newFormValues() formValues {
+	return formValues{make(http.Header)}
+}
+
+func (f formValues) Set(key, value string) formValues {
+	f.Header.Set(key, value)
+	return f
+}
+
+func (f formValues) Add(key, value string) formValues {
+	f.Header.Add(key, value)
+	return f
+}
+
+func (f formValues) Clone() formValues {
+	return formValues{f.Header.Clone()}
+}
+
 // Test Post Policy parsing and checking conditions
 func TestPostPolicyForm(t *testing.T) {
 	pp := minio.NewPostPolicy()
@@ -85,76 +107,193 @@ func TestPostPolicyForm(t *testing.T) {
 	pp.SetContentType("image/jpeg")
 	pp.SetUserMetadata("uuid", "14365123651274")
 	pp.SetKeyStartsWith("user/user1/filename")
-	pp.SetContentLengthRange(1048579, 10485760)
+	pp.SetContentLengthRange(100, 999999) // not testable from this layer, condition is checked in the API handler.
 	pp.SetSuccessStatusAction("201")
+	pp.SetCondition("eq", "X-Amz-Credential", "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request")
+	pp.SetCondition("eq", "X-Amz-Algorithm", "AWS4-HMAC-SHA256")
+	pp.SetCondition("eq", xhttp.AmzDate, "20160727T000000Z")
+
+	defaultFormVals := newFormValues()
+	defaultFormVals.Set("Bucket", "testbucket")
+	defaultFormVals.Set("Content-Type", "image/jpeg")
+	defaultFormVals.Set(xhttp.AmzMetaUUID, "14365123651274")
+	defaultFormVals.Set("Key", "user/user1/filename/${filename}/myfile.txt")
+	defaultFormVals.Set("X-Amz-Credential", "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request")
+	defaultFormVals.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256")
+	defaultFormVals.Set(xhttp.AmzDate, "20160727T000000Z")
+	defaultFormVals.Set("Success_action_status", "201")
+
+	policyCondFailedErr := "Invalid according to Policy: Policy Condition failed"
 
 	type testCase struct {
-		Bucket              string
-		Key                 string
-		XAmzDate            string
-		XAmzAlgorithm       string
-		XAmzCredential      string
-		XAmzMetaUUID        string
-		ContentType         string
-		SuccessActionStatus string
-		Policy              string
-		Expired             bool
-		expectedErr         error
+		name    string
+		fv      formValues
+		expired bool
+		wantErr string
 	}
 
+	// Test case just contains fields we override from defaultFormVals.
 	testCases := []testCase{
-		// Everything is fine with this test
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: nil},
-		// Expired policy document
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", Expired: true, expectedErr: fmt.Errorf("Invalid according to Policy: Policy expired")},
-		// Different AMZ date
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "2017T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
-		// Key which doesn't start with user/user1/filename
-		{Bucket: "testbucket", Key: "myfile.txt", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
-		// Incorrect bucket name.
-		{Bucket: "incorrect", Key: "user/user1/filename/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
-		// Incorrect key name
-		{Bucket: "testbucket", Key: "incorrect", XAmzDate: "20160727T000000Z", XAmzMetaUUID: "14365123651274", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
-		// Incorrect date
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "incorrect", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
-		// Incorrect ContentType
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "14365123651274", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "incorrect", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed")},
-		// Incorrect Metadata
-		{Bucket: "testbucket", Key: "user/user1/filename/${filename}/myfile.txt", XAmzMetaUUID: "151274", SuccessActionStatus: "201", XAmzCredential: "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request", XAmzDate: "20160727T000000Z", XAmzAlgorithm: "AWS4-HMAC-SHA256", ContentType: "image/jpeg", expectedErr: fmt.Errorf("Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]")},
+		{
+			name:    "happy path no errors",
+			fv:      defaultFormVals.Clone(),
+			wantErr: "",
+		},
+		{
+			name:    "expired policy document",
+			fv:      defaultFormVals.Clone(),
+			expired: true,
+			wantErr: "Invalid according to Policy: Policy expired",
+		},
+		{
+			name:    "different AMZ date",
+			fv:      defaultFormVals.Clone().Set(xhttp.AmzDate, "2017T000000Z"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect date",
+			fv:      defaultFormVals.Clone().Set(xhttp.AmzDate, "incorrect"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "key which doesn't start with user/user1/filename",
+			fv:      defaultFormVals.Clone().Set("Key", "myfile.txt"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect key name",
+			fv:      defaultFormVals.Clone().Set("Key", "incorrect"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect bucket name",
+			fv:      defaultFormVals.Clone().Set("Bucket", "incorrect"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect ContentType",
+			fv:      defaultFormVals.Clone().Set(xhttp.ContentType, "incorrect"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect X-Amz-Algorithm",
+			fv:      defaultFormVals.Clone().Set(xhttp.AmzAlgorithm, "incorrect"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect X-Amz-Credential",
+			fv:      defaultFormVals.Clone().Set(xhttp.AmzCredential, "incorrect"),
+			wantErr: policyCondFailedErr,
+		},
+		{
+			name:    "incorrect metadata uuid",
+			fv:      defaultFormVals.Clone().Set(xhttp.AmzMetaUUID, "151274"),
+			wantErr: "Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]",
+		},
+		{
+			name:    "unknown key XAmzMetaName is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(xhttp.AmzMetaName, "my-name"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Meta-Name" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key XAmzChecksumAlgo is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumAlgo), "algo-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Algorithm" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key XAmzChecksumCRC32 is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32), "crc32-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Crc32" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key XAmzChecksumCRC32C is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32C), "crc32c-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Crc32c" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key XAmzChecksumSHA1 is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumSHA1), "sha1-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Sha1" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key XAmzChecksumSHA256 is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumSHA256), "sha256-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Sha256" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key XAmzChecksumMode is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumMode), "mode-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Mode" not specified in the policy.`,
+		},
+		{
+			name:    "unknown key Content-Encoding is error as it does not appear in policy",
+			fv:      defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.ContentEncoding), "encoding-val"),
+			wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "Content-Encoding" not specified in the policy.`,
+		},
+		{
+			name:    "many bucket values",
+			fv:      defaultFormVals.Clone().Add("Bucket", "anotherbucket"),
+			wantErr: "Invalid according to Policy: Policy Condition failed: [eq, $bucket, testbucket]. FormValues have multiple values: [testbucket, anotherbucket]",
+		},
+		{
+			name: "XAmzSignature does not have to appear in policy",
+			fv:   defaultFormVals.Clone().Set(xhttp.AmzSignature, "my-signature"),
+		},
+		{
+			name: "XIgnoreFoo does not have to appear in policy",
+			fv:   defaultFormVals.Clone().Set("X-Ignore-Foo", "my-foo-value"),
+		},
+		{
+			name: "File does not have to appear in policy",
+			fv:   defaultFormVals.Clone().Set("File", "file-value"),
+		},
+		{
+			name: "Signature does not have to appear in policy",
+			fv:   defaultFormVals.Clone().Set(xhttp.AmzSignatureV2, "signature-value"),
+		},
+		{
+			name: "AWSAccessKeyID does not have to appear in policy",
+			fv:   defaultFormVals.Clone().Set(xhttp.AmzAccessKeyID, "access").Set(xhttp.AmzSignatureV2, "signature-value"),
+		},
+		{
+			name: "any form value starting with X-Amz-Server-Side-Encryption- does not have to appear in policy",
+			fv: defaultFormVals.Clone().
+				Set(xhttp.AmzServerSideEncryptionKmsContext, "context-val").
+				Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, "algo-val"),
+		},
 	}
-	// Validate all the test cases.
-	for i, tt := range testCases {
-		formValues := make(http.Header)
-		formValues.Set("Bucket", tt.Bucket)
-		formValues.Set("Key", tt.Key)
-		formValues.Set("Content-Type", tt.ContentType)
-		formValues.Set("X-Amz-Date", tt.XAmzDate)
-		formValues.Set("X-Amz-Meta-Uuid", tt.XAmzMetaUUID)
-		formValues.Set("X-Amz-Algorithm", tt.XAmzAlgorithm)
-		formValues.Set("X-Amz-Credential", tt.XAmzCredential)
-		if tt.Expired {
-			// Expired already.
-			pp.SetExpires(UTCNow().AddDate(0, 0, -10))
-		} else {
-			// Expires in 10 days.
-			pp.SetExpires(UTCNow().AddDate(0, 0, 10))
-		}
 
-		formValues.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String())))
-		formValues.Set("Success_action_status", tt.SuccessActionStatus)
-		policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String())))
-		if err != nil {
-			t.Fatal(err)
-		}
+	// Run tests
+	for _, tt := range testCases {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.expired {
+				// Expired already.
+				pp.SetExpires(UTCNow().AddDate(0, 0, -10))
+			} else {
+				// Expires in 10 days.
+				pp.SetExpires(UTCNow().AddDate(0, 0, 10))
+			}
 
-		postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes))
-		if err != nil {
-			t.Fatal(err)
-		}
+			tt.fv.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String())))
 
-		err = checkPostPolicy(formValues, postPolicyForm)
-		if err != nil && tt.expectedErr != nil && err.Error() != tt.expectedErr.Error() {
-			t.Fatalf("Test %d:, Expected %s, got %s", i+1, tt.expectedErr.Error(), err.Error())
-		}
+			policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String())))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			errStr := ""
+			err = checkPostPolicy(tt.fv.Header, postPolicyForm)
+			if err != nil {
+				errStr = err.Error()
+			}
+			if errStr != tt.wantErr {
+				t.Errorf("test: '%s', want error: '%s', got error: '%s'", tt.name, tt.wantErr, errStr)
+			}
+		})
 	}
 }

cmd/server-main.go

@@ -31,6 +31,7 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strings"
 	"syscall"
 	"time"
@@ -53,7 +54,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/certs"
 	"github.com/minio/pkg/v3/env"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 )
 
@@ -421,6 +421,7 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 		Interface:   ctxt.Interface,
 		SendBufSize: ctxt.SendBufSize,
 		RecvBufSize: ctxt.RecvBufSize,
+		IdleTimeout: ctxt.IdleTimeout,
 	}
 
 	// allow transport to be HTTP/1.1 for proxying.
@@ -878,6 +879,8 @@ func serverMain(ctx *cli.Context) {
 			UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 			UseTLSConfig(newTLSConfig(getCert)).
 			UseIdleTimeout(globalServerCtxt.IdleTimeout).
+			UseReadTimeout(globalServerCtxt.IdleTimeout).
+			UseWriteTimeout(globalServerCtxt.IdleTimeout).
 			UseReadHeaderTimeout(globalServerCtxt.ReadHeaderTimeout).
 			UseBaseContext(GlobalContext).
 			UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
@@ -1104,6 +1107,11 @@ func serverMain(ctx *cli.Context) {
 		// Initialize batch job pool.
 		bootstrapTrace("newBatchJobPool", func() {
 			globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
+			globalBatchJobsMetrics = batchJobMetrics{
+				metrics: make(map[string]*batchJobInfo),
+			}
+			go globalBatchJobsMetrics.init(GlobalContext, newObject)
+			go globalBatchJobsMetrics.purgeJobMetrics()
 		})
 
 		// Prints the formatted startup message, if err is not nil then it prints additional information as well.

cmd/sftp-server-driver.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -45,6 +46,7 @@ const ftpMaxWriteOffset = 100 << 20
 type sftpDriver struct {
 	permissions *ssh.Permissions
 	endpoint    string
+	remoteIP    string
 }
 
 //msgp:ignore sftpMetrics
@@ -89,8 +91,12 @@ func (m *sftpMetrics) log(s *sftp.Request, user string) func(sz int64, err error
 // - sftp.Filewrite
 // - sftp.Filelist
 // - sftp.Filecmd
-func NewSFTPDriver(perms *ssh.Permissions) sftp.Handlers {
-	handler := &sftpDriver{endpoint: fmt.Sprintf("127.0.0.1:%s", globalMinioPort), permissions: perms}
+func NewSFTPDriver(perms *ssh.Permissions, remoteIP string) sftp.Handlers {
+	handler := &sftpDriver{
+		endpoint:    fmt.Sprintf("127.0.0.1:%s", globalMinioPort),
+		permissions: perms,
+		remoteIP:    remoteIP,
+	}
 	return sftp.Handlers{
 		FileGet:  handler,
 		FilePut:  handler,
@@ -99,16 +105,31 @@ func NewSFTPDriver(perms *ssh.Permissions) sftp.Handlers {
 	}
 }
 
+type forwardForTransport struct {
+	tr  http.RoundTripper
+	fwd string
+}
+
+func (f forwardForTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	r.Header.Set("X-Forwarded-For", f.fwd)
+	return f.tr.RoundTrip(r)
+}
+
 func (f *sftpDriver) getMinIOClient() (*minio.Client, error) {
 	mcreds := credentials.NewStaticV4(
 		f.permissions.CriticalOptions["AccessKey"],
 		f.permissions.CriticalOptions["SecretKey"],
 		f.permissions.CriticalOptions["SessionToken"],
 	)
+	// Set X-Forwarded-For on all requests.
+	tr := http.RoundTripper(globalRemoteFTPClientTransport)
+	if f.remoteIP != "" {
+		tr = forwardForTransport{tr: tr, fwd: f.remoteIP}
+	}
 	return minio.New(f.endpoint, &minio.Options{
 		Creds:     mcreds,
 		Secure:    globalIsTLS,
-		Transport: globalRemoteFTPClientTransport,
+		Transport: tr,
 	})
 }
 

cmd/sftp-server.go

@@ -488,7 +488,12 @@ func startSFTPServer(args []string) {
 	sshConfig.AddHostKey(private)
 
 	handleSFTPSession := func(channel ssh.Channel, sconn *ssh.ServerConn) {
-		server := sftp.NewRequestServer(channel, NewSFTPDriver(sconn.Permissions), sftp.WithRSAllocator())
+		var remoteIP string
+
+		if host, _, err := net.SplitHostPort(sconn.RemoteAddr().String()); err == nil {
+			remoteIP = host
+		}
+		server := sftp.NewRequestServer(channel, NewSFTPDriver(sconn.Permissions, remoteIP), sftp.WithRSAllocator())
 		defer server.Close()
 		server.Serve()
 	}

cmd/signature-v4-utils.go

@@ -23,6 +23,7 @@ import (
 	"encoding/hex"
 	"io"
 	"net/http"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -31,7 +32,6 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
-	"golang.org/x/exp/slices"
 )
 
 // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the

cmd/signature-v4.go

@@ -60,7 +60,7 @@ const (
 // getCanonicalHeaders generate a list of request headers with their values
 func getCanonicalHeaders(signedHeaders http.Header) string {
 	var headers []string
-	vals := make(http.Header)
+	vals := make(http.Header, len(signedHeaders))
 	for k, vv := range signedHeaders {
 		k = strings.ToLower(k)
 		headers = append(headers, k)

cmd/storage-datatypes.go

@@ -25,6 +25,8 @@ import (
 	xioutil "github.com/minio/minio/internal/ioutil"
 )
 
+//msgp:clearomitted
+
 //go:generate msgp -file=$GOFILE
 
 // DeleteOptions represents the disk level delete options available for the APIs

cmd/storage-datatypes_gen.go

@@ -746,6 +746,8 @@ func (z *DeleteOptions) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
+	var zb0001Mask uint8 /* 1 bits */
+	_ = zb0001Mask
 	for zb0001 > 0 {
 		zb0001--
 		field, err = dc.ReadMapKeyPtr()
@@ -801,6 +803,7 @@ func (z *DeleteOptions) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "OldDataDir")
 				return
 			}
+			zb0001Mask |= 0x1
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -809,6 +812,11 @@ func (z *DeleteOptions) DecodeMsg(dc *msgp.Reader) (err error) {
 			}
 		}
 	}
+	// Clear omitted fields.
+	if (zb0001Mask & 0x1) == 0 {
+		z.OldDataDir = ""
+	}
+
 	return
 }
 
@@ -827,61 +835,62 @@ func (z *DeleteOptions) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	if zb0001Len == 0 {
-		return
-	}
-	// write "BaseOptions"
-	err = en.Append(0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73)
-	if err != nil {
-		return
-	}
-	// map header, size 0
-	_ = z.BaseOptions
-	err = en.Append(0x80)
-	if err != nil {
-		return
-	}
-	// write "r"
-	err = en.Append(0xa1, 0x72)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.Recursive)
-	if err != nil {
-		err = msgp.WrapError(err, "Recursive")
-		return
-	}
-	// write "i"
-	err = en.Append(0xa1, 0x69)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.Immediate)
-	if err != nil {
-		err = msgp.WrapError(err, "Immediate")
-		return
-	}
-	// write "u"
-	err = en.Append(0xa1, 0x75)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.UndoWrite)
-	if err != nil {
-		err = msgp.WrapError(err, "UndoWrite")
-		return
-	}
-	if (zb0001Mask & 0x10) == 0 { // if not omitted
-		// write "o"
-		err = en.Append(0xa1, 0x6f)
+
+	// skip if no fields are to be emitted
+	if zb0001Len != 0 {
+		// write "BaseOptions"
+		err = en.Append(0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73)
 		if err != nil {
 			return
 		}
-		err = en.WriteString(z.OldDataDir)
+		// map header, size 0
+		_ = z.BaseOptions
+		err = en.Append(0x80)
 		if err != nil {
-			err = msgp.WrapError(err, "OldDataDir")
 			return
 		}
+		// write "r"
+		err = en.Append(0xa1, 0x72)
+		if err != nil {
+			return
+		}
+		err = en.WriteBool(z.Recursive)
+		if err != nil {
+			err = msgp.WrapError(err, "Recursive")
+			return
+		}
+		// write "i"
+		err = en.Append(0xa1, 0x69)
+		if err != nil {
+			return
+		}
+		err = en.WriteBool(z.Immediate)
+		if err != nil {
+			err = msgp.WrapError(err, "Immediate")
+			return
+		}
+		// write "u"
+		err = en.Append(0xa1, 0x75)
+		if err != nil {
+			return
+		}
+		err = en.WriteBool(z.UndoWrite)
+		if err != nil {
+			err = msgp.WrapError(err, "UndoWrite")
+			return
+		}
+		if (zb0001Mask & 0x10) == 0 { // if not omitted
+			// write "o"
+			err = en.Append(0xa1, 0x6f)
+			if err != nil {
+				return
+			}
+			err = en.WriteString(z.OldDataDir)
+			if err != nil {
+				err = msgp.WrapError(err, "OldDataDir")
+				return
+			}
+		}
 	}
 	return
 }
@@ -899,27 +908,28 @@ func (z *DeleteOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
-	if zb0001Len == 0 {
-		return
-	}
-	// string "BaseOptions"
-	o = append(o, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73)
-	// map header, size 0
-	_ = z.BaseOptions
-	o = append(o, 0x80)
-	// string "r"
-	o = append(o, 0xa1, 0x72)
-	o = msgp.AppendBool(o, z.Recursive)
-	// string "i"
-	o = append(o, 0xa1, 0x69)
-	o = msgp.AppendBool(o, z.Immediate)
-	// string "u"
-	o = append(o, 0xa1, 0x75)
-	o = msgp.AppendBool(o, z.UndoWrite)
-	if (zb0001Mask & 0x10) == 0 { // if not omitted
-		// string "o"
-		o = append(o, 0xa1, 0x6f)
-		o = msgp.AppendString(o, z.OldDataDir)
+
+	// skip if no fields are to be emitted
+	if zb0001Len != 0 {
+		// string "BaseOptions"
+		o = append(o, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73)
+		// map header, size 0
+		_ = z.BaseOptions
+		o = append(o, 0x80)
+		// string "r"
+		o = append(o, 0xa1, 0x72)
+		o = msgp.AppendBool(o, z.Recursive)
+		// string "i"
+		o = append(o, 0xa1, 0x69)
+		o = msgp.AppendBool(o, z.Immediate)
+		// string "u"
+		o = append(o, 0xa1, 0x75)
+		o = msgp.AppendBool(o, z.UndoWrite)
+		if (zb0001Mask & 0x10) == 0 { // if not omitted
+			// string "o"
+			o = append(o, 0xa1, 0x6f)
+			o = msgp.AppendString(o, z.OldDataDir)
+		}
 	}
 	return
 }
@@ -934,6 +944,8 @@ func (z *DeleteOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
+	var zb0001Mask uint8 /* 1 bits */
+	_ = zb0001Mask
 	for zb0001 > 0 {
 		zb0001--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
@@ -989,6 +1001,7 @@ func (z *DeleteOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "OldDataDir")
 				return
 			}
+			zb0001Mask |= 0x1
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -997,6 +1010,11 @@ func (z *DeleteOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			}
 		}
 	}
+	// Clear omitted fields.
+	if (zb0001Mask & 0x1) == 0 {
+		z.OldDataDir = ""
+	}
+
 	o = bts
 	return
 }
@@ -2453,6 +2471,9 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 			err = msgp.WrapError(err, "Data")
 			return
 		}
+		if z.Data == nil {
+			z.Data = make([]byte, 0)
+		}
 	}
 	z.NumVersions, err = dc.ReadInt()
 	if err != nil {
@@ -2487,6 +2508,9 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 			err = msgp.WrapError(err, "Checksum")
 			return
 		}
+		if z.Checksum == nil {
+			z.Checksum = make([]byte, 0)
+		}
 	}
 	z.Versioned, err = dc.ReadBool()
 	if err != nil {
@@ -2907,6 +2931,9 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			err = msgp.WrapError(err, "Data")
 			return
 		}
+		if z.Data == nil {
+			z.Data = make([]byte, 0)
+		}
 	}
 	z.NumVersions, bts, err = msgp.ReadIntBytes(bts)
 	if err != nil {
@@ -2937,6 +2964,9 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			err = msgp.WrapError(err, "Checksum")
 			return
 		}
+		if z.Checksum == nil {
+			z.Checksum = make([]byte, 0)
+		}
 	}
 	z.Versioned, bts, err = msgp.ReadBoolBytes(bts)
 	if err != nil {
@@ -3961,6 +3991,9 @@ func (z *RawFileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 					err = msgp.WrapError(err, "Buf")
 					return
 				}
+				if z.Buf == nil {
+					z.Buf = make([]byte, 0)
+				}
 			}
 		default:
 			err = dc.Skip()
@@ -4038,6 +4071,9 @@ func (z *RawFileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					err = msgp.WrapError(err, "Buf")
 					return
 				}
+				if z.Buf == nil {
+					z.Buf = make([]byte, 0)
+				}
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -4220,6 +4256,8 @@ func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
+	var zb0001Mask uint8 /* 1 bits */
+	_ = zb0001Mask
 	for zb0001 > 0 {
 		zb0001--
 		field, err = dc.ReadMapKeyPtr()
@@ -4240,6 +4278,7 @@ func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
+			zb0001Mask |= 0x1
 		case "fl":
 			var zb0002 uint32
 			zb0002, err = dc.ReadArrayHeader()
@@ -4291,6 +4330,11 @@ func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) {
 			}
 		}
 	}
+	// Clear omitted fields.
+	if (zb0001Mask & 0x1) == 0 {
+		z.Prefix = ""
+	}
+
 	return
 }
 
@@ -4309,87 +4353,88 @@ func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	if zb0001Len == 0 {
-		return
-	}
-	// write "bk"
-	err = en.Append(0xa2, 0x62, 0x6b)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Bucket)
-	if err != nil {
-		err = msgp.WrapError(err, "Bucket")
-		return
-	}
-	if (zb0001Mask & 0x2) == 0 { // if not omitted
-		// write "pr"
-		err = en.Append(0xa2, 0x70, 0x72)
+
+	// skip if no fields are to be emitted
+	if zb0001Len != 0 {
+		// write "bk"
+		err = en.Append(0xa2, 0x62, 0x6b)
 		if err != nil {
 			return
 		}
-		err = en.WriteString(z.Prefix)
+		err = en.WriteString(z.Bucket)
 		if err != nil {
-			err = msgp.WrapError(err, "Prefix")
+			err = msgp.WrapError(err, "Bucket")
 			return
 		}
-	}
-	// write "fl"
-	err = en.Append(0xa2, 0x66, 0x6c)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.Files)))
-	if err != nil {
-		err = msgp.WrapError(err, "Files")
-		return
-	}
-	for za0001 := range z.Files {
-		err = en.WriteString(z.Files[za0001])
+		if (zb0001Mask & 0x2) == 0 { // if not omitted
+			// write "pr"
+			err = en.Append(0xa2, 0x70, 0x72)
+			if err != nil {
+				return
+			}
+			err = en.WriteString(z.Prefix)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		}
+		// write "fl"
+		err = en.Append(0xa2, 0x66, 0x6c)
 		if err != nil {
-			err = msgp.WrapError(err, "Files", za0001)
 			return
 		}
-	}
-	// write "ms"
-	err = en.Append(0xa2, 0x6d, 0x73)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.MaxSize)
-	if err != nil {
-		err = msgp.WrapError(err, "MaxSize")
-		return
-	}
-	// write "mo"
-	err = en.Append(0xa2, 0x6d, 0x6f)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.MetadataOnly)
-	if err != nil {
-		err = msgp.WrapError(err, "MetadataOnly")
-		return
-	}
-	// write "ab"
-	err = en.Append(0xa2, 0x61, 0x62)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.AbortOn404)
-	if err != nil {
-		err = msgp.WrapError(err, "AbortOn404")
-		return
-	}
-	// write "mr"
-	err = en.Append(0xa2, 0x6d, 0x72)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.MaxResults)
-	if err != nil {
-		err = msgp.WrapError(err, "MaxResults")
-		return
+		err = en.WriteArrayHeader(uint32(len(z.Files)))
+		if err != nil {
+			err = msgp.WrapError(err, "Files")
+			return
+		}
+		for za0001 := range z.Files {
+			err = en.WriteString(z.Files[za0001])
+			if err != nil {
+				err = msgp.WrapError(err, "Files", za0001)
+				return
+			}
+		}
+		// write "ms"
+		err = en.Append(0xa2, 0x6d, 0x73)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt64(z.MaxSize)
+		if err != nil {
+			err = msgp.WrapError(err, "MaxSize")
+			return
+		}
+		// write "mo"
+		err = en.Append(0xa2, 0x6d, 0x6f)
+		if err != nil {
+			return
+		}
+		err = en.WriteBool(z.MetadataOnly)
+		if err != nil {
+			err = msgp.WrapError(err, "MetadataOnly")
+			return
+		}
+		// write "ab"
+		err = en.Append(0xa2, 0x61, 0x62)
+		if err != nil {
+			return
+		}
+		err = en.WriteBool(z.AbortOn404)
+		if err != nil {
+			err = msgp.WrapError(err, "AbortOn404")
+			return
+		}
+		// write "mr"
+		err = en.Append(0xa2, 0x6d, 0x72)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt(z.MaxResults)
+		if err != nil {
+			err = msgp.WrapError(err, "MaxResults")
+			return
+		}
 	}
 	return
 }
@@ -4407,35 +4452,36 @@ func (z *ReadMultipleReq) MarshalMsg(b []byte) (o []byte, err error) {
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
-	if zb0001Len == 0 {
-		return
+
+	// skip if no fields are to be emitted
+	if zb0001Len != 0 {
+		// string "bk"
+		o = append(o, 0xa2, 0x62, 0x6b)
+		o = msgp.AppendString(o, z.Bucket)
+		if (zb0001Mask & 0x2) == 0 { // if not omitted
+			// string "pr"
+			o = append(o, 0xa2, 0x70, 0x72)
+			o = msgp.AppendString(o, z.Prefix)
+		}
+		// string "fl"
+		o = append(o, 0xa2, 0x66, 0x6c)
+		o = msgp.AppendArrayHeader(o, uint32(len(z.Files)))
+		for za0001 := range z.Files {
+			o = msgp.AppendString(o, z.Files[za0001])
+		}
+		// string "ms"
+		o = append(o, 0xa2, 0x6d, 0x73)
+		o = msgp.AppendInt64(o, z.MaxSize)
+		// string "mo"
+		o = append(o, 0xa2, 0x6d, 0x6f)
+		o = msgp.AppendBool(o, z.MetadataOnly)
+		// string "ab"
+		o = append(o, 0xa2, 0x61, 0x62)
+		o = msgp.AppendBool(o, z.AbortOn404)
+		// string "mr"
+		o = append(o, 0xa2, 0x6d, 0x72)
+		o = msgp.AppendInt(o, z.MaxResults)
 	}
-	// string "bk"
-	o = append(o, 0xa2, 0x62, 0x6b)
-	o = msgp.AppendString(o, z.Bucket)
-	if (zb0001Mask & 0x2) == 0 { // if not omitted
-		// string "pr"
-		o = append(o, 0xa2, 0x70, 0x72)
-		o = msgp.AppendString(o, z.Prefix)
-	}
-	// string "fl"
-	o = append(o, 0xa2, 0x66, 0x6c)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.Files)))
-	for za0001 := range z.Files {
-		o = msgp.AppendString(o, z.Files[za0001])
-	}
-	// string "ms"
-	o = append(o, 0xa2, 0x6d, 0x73)
-	o = msgp.AppendInt64(o, z.MaxSize)
-	// string "mo"
-	o = append(o, 0xa2, 0x6d, 0x6f)
-	o = msgp.AppendBool(o, z.MetadataOnly)
-	// string "ab"
-	o = append(o, 0xa2, 0x61, 0x62)
-	o = msgp.AppendBool(o, z.AbortOn404)
-	// string "mr"
-	o = append(o, 0xa2, 0x6d, 0x72)
-	o = msgp.AppendInt(o, z.MaxResults)
 	return
 }
 
@@ -4449,6 +4495,8 @@ func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
+	var zb0001Mask uint8 /* 1 bits */
+	_ = zb0001Mask
 	for zb0001 > 0 {
 		zb0001--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
@@ -4469,6 +4517,7 @@ func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
+			zb0001Mask |= 0x1
 		case "fl":
 			var zb0002 uint32
 			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -4520,6 +4569,11 @@ func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			}
 		}
 	}
+	// Clear omitted fields.
+	if (zb0001Mask & 0x1) == 0 {
+		z.Prefix = ""
+	}
+
 	o = bts
 	return
 }
@@ -4544,6 +4598,8 @@ func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
+	var zb0001Mask uint8 /* 2 bits */
+	_ = zb0001Mask
 	for zb0001 > 0 {
 		zb0001--
 		field, err = dc.ReadMapKeyPtr()
@@ -4564,6 +4620,7 @@ func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
+			zb0001Mask |= 0x1
 		case "fl":
 			z.File, err = dc.ReadString()
 			if err != nil {
@@ -4582,6 +4639,7 @@ func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Error")
 				return
 			}
+			zb0001Mask |= 0x2
 		case "d":
 			z.Data, err = dc.ReadBytes(z.Data)
 			if err != nil {
@@ -4602,6 +4660,15 @@ func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) {
 			}
 		}
 	}
+	// Clear omitted fields.
+	if zb0001Mask != 0x3 {
+		if (zb0001Mask & 0x1) == 0 {
+			z.Prefix = ""
+		}
+		if (zb0001Mask & 0x2) == 0 {
+			z.Error = ""
+		}
+	}
 	return
 }
 
@@ -4624,82 +4691,83 @@ func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	if zb0001Len == 0 {
-		return
-	}
-	// write "bk"
-	err = en.Append(0xa2, 0x62, 0x6b)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Bucket)
-	if err != nil {
-		err = msgp.WrapError(err, "Bucket")
-		return
-	}
-	if (zb0001Mask & 0x2) == 0 { // if not omitted
-		// write "pr"
-		err = en.Append(0xa2, 0x70, 0x72)
+
+	// skip if no fields are to be emitted
+	if zb0001Len != 0 {
+		// write "bk"
+		err = en.Append(0xa2, 0x62, 0x6b)
 		if err != nil {
 			return
 		}
-		err = en.WriteString(z.Prefix)
+		err = en.WriteString(z.Bucket)
 		if err != nil {
-			err = msgp.WrapError(err, "Prefix")
+			err = msgp.WrapError(err, "Bucket")
 			return
 		}
-	}
-	// write "fl"
-	err = en.Append(0xa2, 0x66, 0x6c)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.File)
-	if err != nil {
-		err = msgp.WrapError(err, "File")
-		return
-	}
-	// write "ex"
-	err = en.Append(0xa2, 0x65, 0x78)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.Exists)
-	if err != nil {
-		err = msgp.WrapError(err, "Exists")
-		return
-	}
-	if (zb0001Mask & 0x10) == 0 { // if not omitted
-		// write "er"
-		err = en.Append(0xa2, 0x65, 0x72)
+		if (zb0001Mask & 0x2) == 0 { // if not omitted
+			// write "pr"
+			err = en.Append(0xa2, 0x70, 0x72)
+			if err != nil {
+				return
+			}
+			err = en.WriteString(z.Prefix)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		}
+		// write "fl"
+		err = en.Append(0xa2, 0x66, 0x6c)
 		if err != nil {
 			return
 		}
-		err = en.WriteString(z.Error)
+		err = en.WriteString(z.File)
 		if err != nil {
-			err = msgp.WrapError(err, "Error")
+			err = msgp.WrapError(err, "File")
+			return
+		}
+		// write "ex"
+		err = en.Append(0xa2, 0x65, 0x78)
+		if err != nil {
+			return
+		}
+		err = en.WriteBool(z.Exists)
+		if err != nil {
+			err = msgp.WrapError(err, "Exists")
+			return
+		}
+		if (zb0001Mask & 0x10) == 0 { // if not omitted
+			// write "er"
+			err = en.Append(0xa2, 0x65, 0x72)
+			if err != nil {
+				return
+			}
+			err = en.WriteString(z.Error)
+			if err != nil {
+				err = msgp.WrapError(err, "Error")
+				return
+			}
+		}
+		// write "d"
+		err = en.Append(0xa1, 0x64)
+		if err != nil {
+			return
+		}
+		err = en.WriteBytes(z.Data)
+		if err != nil {
+			err = msgp.WrapError(err, "Data")
+			return
+		}
+		// write "m"
+		err = en.Append(0xa1, 0x6d)
+		if err != nil {
+			return
+		}
+		err = en.WriteTime(z.Modtime)
+		if err != nil {
+			err = msgp.WrapError(err, "Modtime")
 			return
 		}
-	}
-	// write "d"
-	err = en.Append(0xa1, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteBytes(z.Data)
-	if err != nil {
-		err = msgp.WrapError(err, "Data")
-		return
-	}
-	// write "m"
-	err = en.Append(0xa1, 0x6d)
-	if err != nil {
-		return
-	}
-	err = en.WriteTime(z.Modtime)
-	if err != nil {
-		err = msgp.WrapError(err, "Modtime")
-		return
 	}
 	return
 }
@@ -4721,34 +4789,35 @@ func (z *ReadMultipleResp) MarshalMsg(b []byte) (o []byte, err error) {
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
-	if zb0001Len == 0 {
-		return
+
+	// skip if no fields are to be emitted
+	if zb0001Len != 0 {
+		// string "bk"
+		o = append(o, 0xa2, 0x62, 0x6b)
+		o = msgp.AppendString(o, z.Bucket)
+		if (zb0001Mask & 0x2) == 0 { // if not omitted
+			// string "pr"
+			o = append(o, 0xa2, 0x70, 0x72)
+			o = msgp.AppendString(o, z.Prefix)
+		}
+		// string "fl"
+		o = append(o, 0xa2, 0x66, 0x6c)
+		o = msgp.AppendString(o, z.File)
+		// string "ex"
+		o = append(o, 0xa2, 0x65, 0x78)
+		o = msgp.AppendBool(o, z.Exists)
+		if (zb0001Mask & 0x10) == 0 { // if not omitted
+			// string "er"
+			o = append(o, 0xa2, 0x65, 0x72)
+			o = msgp.AppendString(o, z.Error)
+		}
+		// string "d"
+		o = append(o, 0xa1, 0x64)
+		o = msgp.AppendBytes(o, z.Data)
+		// string "m"
+		o = append(o, 0xa1, 0x6d)
+		o = msgp.AppendTime(o, z.Modtime)
 	}
-	// string "bk"
-	o = append(o, 0xa2, 0x62, 0x6b)
-	o = msgp.AppendString(o, z.Bucket)
-	if (zb0001Mask & 0x2) == 0 { // if not omitted
-		// string "pr"
-		o = append(o, 0xa2, 0x70, 0x72)
-		o = msgp.AppendString(o, z.Prefix)
-	}
-	// string "fl"
-	o = append(o, 0xa2, 0x66, 0x6c)
-	o = msgp.AppendString(o, z.File)
-	// string "ex"
-	o = append(o, 0xa2, 0x65, 0x78)
-	o = msgp.AppendBool(o, z.Exists)
-	if (zb0001Mask & 0x10) == 0 { // if not omitted
-		// string "er"
-		o = append(o, 0xa2, 0x65, 0x72)
-		o = msgp.AppendString(o, z.Error)
-	}
-	// string "d"
-	o = append(o, 0xa1, 0x64)
-	o = msgp.AppendBytes(o, z.Data)
-	// string "m"
-	o = append(o, 0xa1, 0x6d)
-	o = msgp.AppendTime(o, z.Modtime)
 	return
 }
 
@@ -4762,6 +4831,8 @@ func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
+	var zb0001Mask uint8 /* 2 bits */
+	_ = zb0001Mask
 	for zb0001 > 0 {
 		zb0001--
 		field, bts, err = msgp.ReadMapKeyZC(bts)
@@ -4782,6 +4853,7 @@ func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
+			zb0001Mask |= 0x1
 		case "fl":
 			z.File, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
@@ -4800,6 +4872,7 @@ func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Error")
 				return
 			}
+			zb0001Mask |= 0x2
 		case "d":
 			z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data)
 			if err != nil {
@@ -4820,6 +4893,15 @@ func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			}
 		}
 	}
+	// Clear omitted fields.
+	if zb0001Mask != 0x3 {
+		if (zb0001Mask & 0x1) == 0 {
+			z.Prefix = ""
+		}
+		if (zb0001Mask & 0x2) == 0 {
+			z.Error = ""
+		}
+	}
 	o = bts
 	return
 }

cmd/storage-rest-client.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/gob"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -465,16 +464,20 @@ func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, pa
 // CheckParts - stat all file parts.
 func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
 	var resp *CheckPartsResp
-	resp, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{
+	st, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{
 		DiskID:   *client.diskID.Load(),
 		Volume:   volume,
 		FilePath: path,
 		FI:       fi,
 	})
 	if err != nil {
-		return nil, err
+		return nil, toStorageErr(err)
 	}
-	return resp, nil
+	err = st.Results(func(r *CheckPartsResp) error {
+		resp = r
+		return nil
+	})
+	return resp, toStorageErr(err)
 }
 
 // RenameData - rename source path to destination path atomically, metadata and data file.
@@ -842,12 +845,16 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
 		return nil, toStorageErr(err)
 	}
 
-	verifyResp := &CheckPartsResp{}
-	if err = gob.NewDecoder(respReader).Decode(verifyResp); err != nil {
+	dec := msgpNewReader(respReader)
+	defer readMsgpReaderPoolPut(dec)
+
+	verifyResp := CheckPartsResp{}
+	err = verifyResp.DecodeMsg(dec)
+	if err != nil {
 		return nil, toStorageErr(err)
 	}
 
-	return verifyResp, nil
+	return &verifyResp, nil
 }
 
 func (client *storageRESTClient) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) {

cmd/storage-rest-server.go

@@ -56,7 +56,7 @@ type storageRESTServer struct {
 }
 
 var (
-	storageCheckPartsRPC       = grid.NewSingleHandler[*CheckPartsHandlerParams, *CheckPartsResp](grid.HandlerCheckParts2, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, func() *CheckPartsResp { return &CheckPartsResp{} })
+	storageCheckPartsRPC       = grid.NewStream[*CheckPartsHandlerParams, grid.NoPayload, *CheckPartsResp](grid.HandlerCheckParts3, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, nil, func() *CheckPartsResp { return &CheckPartsResp{} })
 	storageDeleteFileRPC       = grid.NewSingleHandler[*DeleteFileHandlerParams, grid.NoPayload](grid.HandlerDeleteFile, func() *DeleteFileHandlerParams { return &DeleteFileHandlerParams{} }, grid.NewNoPayload).AllowCallRequestPool(true)
 	storageDeleteVersionRPC    = grid.NewSingleHandler[*DeleteVersionHandlerParams, grid.NoPayload](grid.HandlerDeleteVersion, func() *DeleteVersionHandlerParams { return &DeleteVersionHandlerParams{} }, grid.NewNoPayload)
 	storageDiskInfoRPC         = grid.NewSingleHandler[*DiskInfoOptions, *DiskInfo](grid.HandlerDiskInfo, func() *DiskInfoOptions { return &DiskInfoOptions{} }, func() *DiskInfo { return &DiskInfo{} }).WithSharedResponse().AllowCallRequestPool(true)
@@ -456,16 +456,20 @@ func (s *storageRESTServer) UpdateMetadataHandler(p *MetadataHandlerParams) (gri
 	return grid.NewNPErr(s.getStorage().UpdateMetadata(context.Background(), volume, filePath, p.FI, p.UpdateOpts))
 }
 
-// CheckPartsHandler - check if a file metadata exists.
-func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (*CheckPartsResp, *grid.RemoteErr) {
+// CheckPartsHandler - check if a file parts exists.
+func (s *storageRESTServer) CheckPartsHandler(ctx context.Context, p *CheckPartsHandlerParams, out chan<- *CheckPartsResp) *grid.RemoteErr {
 	if !s.checkID(p.DiskID) {
-		return nil, grid.NewRemoteErr(errDiskNotFound)
+		return grid.NewRemoteErr(errDiskNotFound)
 	}
 	volume := p.Volume
 	filePath := p.FilePath
 
-	resp, err := s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI)
-	return resp, grid.NewRemoteErr(err)
+	resp, err := s.getStorage().CheckParts(ctx, volume, filePath, p.FI)
+	if err != nil {
+		return grid.NewRemoteErr(err)
+	}
+	out <- resp
+	return grid.NewRemoteErr(err)
 }
 
 func (s *storageRESTServer) WriteAllHandler(p *WriteAllHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
@@ -1382,7 +1386,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			logger.FatalIf(storageRenameDataRPC.Register(gm, server.RenameDataHandler, endpoint.Path), "unable to register handler")
 			logger.FatalIf(storageRenameDataInlineRPC.Register(gm, server.RenameDataInlineHandler, endpoint.Path), "unable to register handler")
 			logger.FatalIf(storageDeleteFileRPC.Register(gm, server.DeleteFileHandler, endpoint.Path), "unable to register handler")
-			logger.FatalIf(storageCheckPartsRPC.Register(gm, server.CheckPartsHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageCheckPartsRPC.RegisterNoInput(gm, server.CheckPartsHandler, endpoint.Path), "unable to register handler")
 			logger.FatalIf(storageReadVersionRPC.Register(gm, server.ReadVersionHandlerWS, endpoint.Path), "unable to register handler")
 			logger.FatalIf(storageWriteMetadataRPC.Register(gm, server.WriteMetadataHandler, endpoint.Path), "unable to register handler")
 			logger.FatalIf(storageUpdateMetadataRPC.Register(gm, server.UpdateMetadataHandler, endpoint.Path), "unable to register handler")

cmd/sts-handlers_test.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"os"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -34,7 +35,6 @@ import (
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/pkg/v3/ldap"
-	"golang.org/x/exp/slices"
 )
 
 func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
@@ -740,6 +740,7 @@ func TestIAMWithLDAPServerSuite(t *testing.T) {
 				suite.TestLDAPSTSServiceAccountsWithGroups(c)
 				suite.TestLDAPAttributesLookup(c)
 				suite.TestLDAPCyrillicUser(c)
+				suite.TestLDAPSlashDN(c)
 				suite.TearDownSuite(c)
 			},
 		)
@@ -770,6 +771,7 @@ func TestIAMWithLDAPNonNormalizedBaseDNConfigServerSuite(t *testing.T) {
 				suite.TestLDAPSTSServiceAccounts(c)
 				suite.TestLDAPSTSServiceAccountsWithUsername(c)
 				suite.TestLDAPSTSServiceAccountsWithGroups(c)
+				suite.TestLDAPSlashDN(c)
 				suite.TearDownSuite(c)
 			},
 		)
@@ -1997,7 +1999,7 @@ func (s *TestSuiteIAM) TestLDAPCyrillicUser(c *check) {
 		}
 
 		// Validate claims.
-		dnClaim := claims[ldapActualUser].(string)
+		dnClaim := claims.MapClaims[ldapActualUser].(string)
 		if dnClaim != testCase.dn {
 			c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim)
 		}
@@ -2008,6 +2010,93 @@ func (s *TestSuiteIAM) TestLDAPCyrillicUser(c *check) {
 	}
 }
 
+func (s *TestSuiteIAM) TestLDAPSlashDN(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	policyReq := madmin.PolicyAssociationReq{
+		Policies: []string{"readwrite"},
+	}
+
+	cases := []struct {
+		username string
+		dn       string
+		group    string
+	}{
+		{
+			username: "slashuser",
+			dn:       "uid=slash/user,ou=people,ou=swengg,dc=min,dc=io",
+		},
+		{
+			username: "dillon",
+			dn:       "uid=dillon,ou=people,ou=swengg,dc=min,dc=io",
+			group:    "cn=project/d,ou=groups,ou=swengg,dc=min,dc=io",
+		},
+	}
+
+	conn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
+	if err != nil {
+		c.Fatalf("LDAP connect failed: %v", err)
+	}
+	defer conn.Close()
+
+	for i, testCase := range cases {
+		if testCase.group != "" {
+			policyReq.Group = testCase.group
+			policyReq.User = ""
+		} else {
+			policyReq.User = testCase.dn
+			policyReq.Group = ""
+		}
+
+		if _, err := s.adm.AttachPolicyLDAP(ctx, policyReq); err != nil {
+			c.Fatalf("Unable to attach  policy: %v", err)
+		}
+
+		ldapID := cr.LDAPIdentity{
+			Client:       s.TestSuiteCommon.client,
+			STSEndpoint:  s.endPoint,
+			LDAPUsername: testCase.username,
+			LDAPPassword: testCase.username,
+		}
+
+		value, err := ldapID.Retrieve()
+		if err != nil {
+			c.Fatalf("Expected to generate STS creds, got err: %#v", err)
+		}
+
+		// Retrieve the STS account's credential object.
+		u, ok := globalIAMSys.GetUser(ctx, value.AccessKeyID)
+		if !ok {
+			c.Fatalf("Expected to find user %s", value.AccessKeyID)
+		}
+
+		if u.Credentials.AccessKey != value.AccessKeyID {
+			c.Fatalf("Expected access key %s, got %s", value.AccessKeyID, u.Credentials.AccessKey)
+		}
+
+		// Retrieve the credential's claims.
+		secret, err := getTokenSigningKey()
+		if err != nil {
+			c.Fatalf("Error getting token signing key: %v", err)
+		}
+		claims, err := getClaimsFromTokenWithSecret(value.SessionToken, secret)
+		if err != nil {
+			c.Fatalf("Error getting claims from token: %v", err)
+		}
+
+		// Validate claims.
+		dnClaim := claims.MapClaims[ldapActualUser].(string)
+		if dnClaim != testCase.dn {
+			c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim)
+		}
+
+		if _, err = s.adm.DetachPolicyLDAP(ctx, policyReq); err != nil {
+			c.Fatalf("Unable to detach user policy: %v", err)
+		}
+	}
+}
+
 func (s *TestSuiteIAM) TestLDAPAttributesLookup(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -2079,11 +2168,11 @@ func (s *TestSuiteIAM) TestLDAPAttributesLookup(c *check) {
 		}
 
 		// Validate claims. Check if the sshPublicKey claim is present.
-		dnClaim := claims[ldapActualUser].(string)
+		dnClaim := claims.MapClaims[ldapActualUser].(string)
 		if dnClaim != testCase.dn {
 			c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim)
 		}
-		sshPublicKeyClaim := claims[ldapAttribPrefix+"sshPublicKey"].([]interface{})[0].(string)
+		sshPublicKeyClaim := claims.MapClaims[ldapAttribPrefix+"sshPublicKey"].([]interface{})[0].(string)
 		if sshPublicKeyClaim == "" {
 			c.Fatalf("Test %d: expected sshPublicKey claim to be present", i+1)
 		}

cmd/test-utils_test.go

@@ -1638,7 +1638,7 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
 		t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Object API Nil Test expected to fail with %d, but failed with %d", accessDenied, rec.Code)))
 	}
 
-	// HEAD HTTTP request doesn't contain response body.
+	// HEAD HTTP request doesn't contain response body.
 	if anonReq.Method != http.MethodHead {
 		// read the response body.
 		var actualContent []byte
@@ -2247,12 +2247,12 @@ func getEndpointsLocalAddr(endpointServerPools EndpointServerPools) string {
 }
 
 // fetches a random number between range min-max.
-func getRandomRange(min, max int, seed int64) int {
+func getRandomRange(minN, maxN int, seed int64) int {
 	// special value -1 means no explicit seeding.
-	if seed != -1 {
-		rand.Seed(seed)
+	if seed == -1 {
+		return rand.New(rand.NewSource(time.Now().UnixNano())).Intn(maxN-minN) + minN
 	}
-	return rand.Intn(max-min) + min
+	return rand.New(rand.NewSource(seed)).Intn(maxN-minN) + minN
 }
 
 // Randomizes the order of bytes in the byte array