Update console to v1.6.0 (#19933)

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

.github/workflows/helm-lint.yml

@@ -22,7 +22,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
 
       - name: Run helm lint
         run: |

.github/workflows/mint.yml

@@ -55,9 +55,10 @@ jobs:
         run: |
           ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
 
-      - name: resiliency
-        run: |
-          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+      # FIXME: renable this back when we have a valid way to add deadlines for PUT()s (internode CreateFile)
+      # - name: resiliency
+      #   run: |
+      #     ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
 
       - name: The job must cleanup
         if: ${{ always() }}

.github/workflows/vulncheck.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.3
+          go-version: 1.22.4
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

Makefile

@@ -47,7 +47,7 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 check: test
 test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue ./...
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue,dev ./...
 
 test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
@@ -84,13 +84,13 @@ test-race: verifiers build ## builds minio, runs linters, tests (race)
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-test-iam: build ## verify IAM (external IDP, etcd backends)
+test-iam: install-race ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue,dev -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue,dev -v -run TestIAM* ./cmd
 
-test-iam-ldap-upgrade-import: build ## verify IAM (external LDAP IDP)
+test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP)
 	@echo "Running upgrade tests for IAM (LDAP backend)"
 	@env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh
 
@@ -106,7 +106,10 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
+test-delete-marker-proxying:
+	@(env bash $(PWD)/docs/bucket/replication/test_del_marker_proxying.sh)
+
+test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error test-delete-marker-proxying ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
 test-site-replication-ldap: install-race ## verify automatic site replication
@@ -193,7 +196,7 @@ docker: build ## builds minio docker container
 
 install-race: checks build-debugging ## builds minio to $(PWD)
 	@echo "Building minio binary with -race to './minio'"
-	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue,dev -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@echo "Installing minio binary with -race to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -af $(PWD)/minio $(GOPATH)/bin/minio
 

buildscripts/minio-upgrade.sh

@@ -4,10 +4,22 @@ trap 'cleanup $LINENO' ERR
 
 # shellcheck disable=SC2120
 cleanup() {
-	MINIO_VERSION=dev docker-compose \
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose \
 		-f "buildscripts/upgrade-tests/compose.yml" \
-		rm -s -f
+		down || true
+
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose \
+		-f "buildscripts/upgrade-tests/compose.yml" \
+		rm || true
+
+	for volume in $(docker volume ls -q | grep upgrade); do
+		docker volume rm ${volume} || true
+	done
+
 	docker volume prune -f
+	docker system prune -f || true
+	docker volume prune -f || true
+	docker volume rm $(docker volume ls -q -f dangling=true) || true
 }
 
 verify_checksum_after_heal() {
@@ -60,6 +72,8 @@ __init__() {
 	go install github.com/docker/compose/v2/cmd@latest
 	mv -v /tmp/gopath/bin/cmd /tmp/gopath/bin/docker-compose
 
+	cleanup
+
 	TAG=minio/minio:dev make docker
 
 	MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
@@ -77,11 +91,11 @@ __init__() {
 
 	curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
 
-	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
 }
 
 main() {
-	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
+	MINIO_VERSION=dev /tmp/gopath/bin/docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
 
 	add_alias
 

buildscripts/upgrade-tests/compose.yml

@@ -1,5 +1,3 @@
-version: '3.7'
-
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
   image: minio/minio:${MINIO_VERSION}

buildscripts/verify-healing-empty-erasure-set.sh

@@ -38,8 +38,40 @@ function start_minio_3_node() {
 	disown $pid3
 
 	export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
+
 	/tmp/mc ready myminio
 
+	# Wait for all drives to be online and formatted
+	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].state | select(. != "ok")' | wc -l) -gt 0 ]; do sleep 1; done
+	# Wait for all drives to be healed
+	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].healing | select(. != null) | select(. == true)' | wc -l) -gt 0 ]; do sleep 1; done
+
+	# Wait for Status: in MinIO output
+	while true; do
+		rv=$(check_online)
+		if [ "$rv" != "1" ]; then
+			# success
+			break
+		fi
+
+		# Check if we should retry
+		retry=$((retry + 1))
+		if [ $retry -le 20 ]; then
+			sleep 5
+			continue
+		fi
+
+		# Failure
+		for i in $(seq 1 3); do
+			echo "server$i log:"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
+		done
+		pkill -9 minio
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	done
+
 	if ! ps -p $pid1 1>&2 >/dev/null; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/dist-minio-server1.log"
@@ -90,7 +122,7 @@ function check_online() {
 }
 
 function purge() {
-	rm -rf "$1"
+	echo rm -rf "$1"
 }
 
 function __init__() {
@@ -117,18 +149,6 @@ function perform_test() {
 
 	set -x
 	start_minio_3_node $2
-
-	rv=$(check_online)
-	if [ "$rv" == "1" ]; then
-		for i in $(seq 1 3); do
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		pkill -9 minio
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
 }
 
 function main() {

cmd/admin-handlers-idp-ldap.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
@@ -284,6 +285,18 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 		opts.claims[ldapUser] = targetUser // DN
 		opts.claims[ldapActualUser] = lookupResult.ActualDN
 
+		// Check if this user or their groups have a policy applied.
+		ldapPolicies, err := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if len(ldapPolicies) == 0 {
+			err = fmt.Errorf("No policy set for user `%s` or any of their groups: `%s`", opts.claims[ldapActualUser], strings.Join(targetGroups, "`,`"))
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL)
+			return
+		}
+
 		// Add LDAP attributes that were looked up into the claims.
 		for attribKey, attribValue := range lookupResult.Attributes {
 			opts.claims[ldapAttribPrefix+attribKey] = attribValue

cmd/api-errors.go

@@ -445,6 +445,8 @@ const (
 	ErrAdminNoAccessKey
 	ErrAdminNoSecretKey
 
+	ErrIAMNotInitialized
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -967,7 +969,7 @@ var errorCodes = errorCodeMap{
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
 		Description:    "Remote service connection error",
-		HTTPStatusCode: http.StatusNotFound,
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrReplicationBandwidthLimitError: {
 		Code:           "XMinioAdminReplicationBandwidthLimitError",
@@ -1305,6 +1307,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Server not initialized yet, please try again.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrIAMNotInitialized: {
+		Code:           "XMinioIAMNotInitialized",
+		Description:    "IAM sub-system not initialized yet, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	ErrBucketMetadataNotInitialized: {
 		Code:           "XMinioBucketMetadataNotInitialized",
 		Description:    "Bucket metadata not initialized yet, please try again.",

cmd/api-response.go

@@ -789,8 +789,8 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 }
 
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
-func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
-	cs := oi.decryptChecksums(0)
+func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo, h http.Header) CompleteMultipartUploadResponse {
+	cs := oi.decryptChecksums(0, h)
 	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,

cmd/background-newdisks-heal-ops.go

@@ -141,14 +141,14 @@ func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	return h
 }
 
-func (h healingTracker) getLastUpdate() time.Time {
+func (h *healingTracker) getLastUpdate() time.Time {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
 	return h.LastUpdate
 }
 
-func (h healingTracker) getBucket() string {
+func (h *healingTracker) getBucket() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
@@ -162,7 +162,7 @@ func (h *healingTracker) setBucket(bucket string) {
 	h.Bucket = bucket
 }
 
-func (h healingTracker) getObject() string {
+func (h *healingTracker) getObject() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
@@ -453,10 +453,6 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 
 	healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 
-	if len(tracker.QueuedBuckets) > 0 {
-		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
-	}
-
 	if serverDebugLog {
 		tracker.printTo(os.Stdout)
 		fmt.Printf("\n")

cmd/batch-expire.go

@@ -432,7 +432,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 					})
 					if err != nil {
 						stopFn(exp, err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
 					} else {
 						stopFn(exp, err)
 						success = true
@@ -464,13 +464,13 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 				copy(toDelCopy, toDel)
 				var failed int
 				errs := r.Expire(ctx, api, vc, toDel)
-				// reslice toDel in preparation for next retry
-				// attempt
+				// reslice toDel in preparation for next retry attempt
 				toDel = toDel[:0]
 				for i, err := range errs {
 					if err != nil {
 						stopFn(toDelCopy[i], err)
-						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
+							err, attempts))
 						failed++
 						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
 							ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)

cmd/batch-handlers.go

@@ -1476,7 +1476,7 @@ func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string
 
 func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
 	// TODO: support custom storage class for remote replication
-	putOpts, err = putReplicationOpts(ctx, "", objInfo)
+	putOpts, err = putReplicationOpts(ctx, "", objInfo, 0)
 	if err != nil {
 		return putOpts, err
 	}
@@ -1633,7 +1633,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	job.ID = fmt.Sprintf("%s:%d", shortuuid.New(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
+	job.ID = fmt.Sprintf("%s%s%d", shortuuid.New(), getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
 

cmd/bucket-handlers.go

@@ -1420,7 +1420,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if formValues.Get(postPolicyBucketTagging) != "" {
 		tags, err := tags.ParseObjectXML(strings.NewReader(formValues.Get(postPolicyBucketTagging)))
 		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 			return
 		}
 		tagsStr := tags.String()

cmd/bucket-handlers_test.go

@@ -32,7 +32,7 @@ import (
 
 // Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestRemoveBucketHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testRemoveBucketHandler, []string{"RemoveBucket"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testRemoveBucketHandler, endpoints: []string{"RemoveBucket"}})
 }
 
 func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -78,7 +78,7 @@ func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, a
 
 // Wrapper for calling GetBucketPolicy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestGetBucketLocationHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testGetBucketLocationHandler, []string{"GetBucketLocation"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testGetBucketLocationHandler, endpoints: []string{"GetBucketLocation"}})
 }
 
 func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -220,7 +220,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 
 // Wrapper for calling HeadBucket HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestHeadBucketHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testHeadBucketHandler, []string{"HeadBucket"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testHeadBucketHandler, endpoints: []string{"HeadBucket"}})
 }
 
 func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -322,7 +322,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 
 // Wrapper for calling TestListMultipartUploadsHandler tests for both Erasure multiple disks and single node setup.
 func TestListMultipartUploadsHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testListMultipartUploadsHandler, []string{"ListMultipartUploads"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListMultipartUploadsHandler, endpoints: []string{"ListMultipartUploads"}})
 }
 
 // testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
@@ -558,7 +558,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 // Wrapper for calling TestListBucketsHandler tests for both Erasure multiple disks and single node setup.
 func TestListBucketsHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testListBucketsHandler, []string{"ListBuckets"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListBucketsHandler, endpoints: []string{"ListBuckets"}})
 }
 
 // testListBucketsHandler - Tests validate listing of buckets.
@@ -649,7 +649,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 
 // Wrapper for calling DeleteMultipleObjects HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testAPIDeleteMultipleObjectsHandler, []string{"DeleteMultipleObjects", "PutBucketPolicy"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIDeleteMultipleObjectsHandler, endpoints: []string{"DeleteMultipleObjects", "PutBucketPolicy"}})
 }
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,

cmd/bucket-lifecycle-handlers_test.go

@@ -29,7 +29,7 @@ import (
 
 // Test S3 Bucket lifecycle APIs with wrong credentials
 func TestBucketLifecycleWrongCredentials(t *testing.T) {
-	ExecObjectLayerAPITest(t, testBucketLifecycleHandlersWrongCredentials, []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketLifecycleHandlersWrongCredentials, endpoints: []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"}})
 }
 
 // Test for authentication
@@ -145,7 +145,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 
 // Test S3 Bucket lifecycle APIs
 func TestBucketLifecycle(t *testing.T) {
-	ExecObjectLayerAPITest(t, testBucketLifecycleHandlers, []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketLifecycleHandlers, endpoints: []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"}})
 }
 
 // Simple tests of bucket lifecycle: PUT, GET, DELETE.

cmd/bucket-lifecycle.go

@@ -72,6 +72,7 @@ func NewLifecycleSys() *LifecycleSys {
 }
 
 func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+	sz, _ := oi.GetActualSize()
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceILM,
 		Time:      startTime,
@@ -79,6 +80,7 @@ func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event
 		FuncName:  event,
 		Duration:  duration,
 		Path:      pathJoin(oi.Bucket, oi.Name),
+		Bytes:     sz,
 		Error:     "",
 		Message:   getSource(4),
 		Custom:    map[string]string{"version-id": oi.VersionID},
@@ -277,6 +279,10 @@ func (es *expiryState) getWorkerCh(h uint64) chan<- expiryOp {
 }
 
 func (es *expiryState) ResizeWorkers(n int) {
+	if n == 0 {
+		n = 100
+	}
+
 	// Lock to avoid multiple resizes to happen at the same time.
 	es.mu.Lock()
 	defer es.mu.Unlock()
@@ -538,6 +544,10 @@ func (t *transitionState) UpdateWorkers(n int) {
 }
 
 func (t *transitionState) updateWorkers(n int) {
+	if n == 0 {
+		n = 100
+	}
+
 	for t.numWorkers < n {
 		go t.worker(t.objAPI)
 		t.numWorkers++
@@ -573,6 +583,10 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
 		switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+			if obj.DeleteMarker || obj.IsDir {
+				// nothing to transition
+				return
+			}
 			globalTransitionState.queueTransitionTask(obj, event, src)
 		}
 	}
@@ -727,7 +741,7 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
 		return nil, fmt.Errorf("transition storage class not configured: %w", err)
 	}
 
-	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	fn, off, length, err := NewGetObjectReader(rs, oi, opts, h)
 	if err != nil {
 		return nil, ErrorRespToObjectError(err, bucket, object)
 	}

cmd/bucket-listobjects-handlers.go

@@ -202,7 +202,7 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re
 
 	if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(prefix, archivePattern) {
 		// Initiate a list objects operation inside a zip file based in the input params
-		listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
+		listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, startAfter, r.Header)
 	} else {
 		// Initiate a list objects operation based on the input params.
 		// On success would return back ListObjectsInfo object to be
@@ -231,7 +231,7 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1
 	}
-	i := strings.Index(token, ":")
+	i := strings.Index(token, getKeySeparator())
 	if i < 0 {
 		return token, -1
 	}

cmd/bucket-policy-handlers_test.go

@@ -107,7 +107,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolic
 
 // Wrapper for calling Create Bucket and ensure we get one and only one success.
 func TestCreateBucket(t *testing.T) {
-	ExecObjectLayerAPITest(t, testCreateBucket, []string{"MakeBucket"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testCreateBucket, endpoints: []string{"MakeBucket"}})
 }
 
 // testCreateBucket - Test for calling Create Bucket and ensure we get one and only one success.
@@ -154,7 +154,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 
 // Wrapper for calling Put Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestPutBucketPolicyHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testPutBucketPolicyHandler, []string{"PutBucketPolicy"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPutBucketPolicyHandler, endpoints: []string{"PutBucketPolicy"}})
 }
 
 // testPutBucketPolicyHandler - Test for Bucket policy end point.
@@ -373,7 +373,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 // Wrapper for calling Get Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestGetBucketPolicyHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testGetBucketPolicyHandler, []string{"PutBucketPolicy", "GetBucketPolicy"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testGetBucketPolicyHandler, endpoints: []string{"PutBucketPolicy", "GetBucketPolicy"}})
 }
 
 // testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
@@ -577,7 +577,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 // Wrapper for calling Delete Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestDeleteBucketPolicyHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testDeleteBucketPolicyHandler, []string{"PutBucketPolicy", "DeleteBucketPolicy"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testDeleteBucketPolicyHandler, endpoints: []string{"PutBucketPolicy", "DeleteBucketPolicy"}})
 }
 
 // testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.

cmd/bucket-replication-utils.go

@@ -534,7 +534,7 @@ func getHealReplicateObjectInfo(oi ObjectInfo, rcfg replicationConfig) Replicate
 	rstate.ReplicateDecisionStr = dsc.String()
 	asz, _ := oi.GetActualSize()
 
-	return ReplicateObjectInfo{
+	r := ReplicateObjectInfo{
 		Name:                       oi.Name,
 		Size:                       oi.Size,
 		ActualSize:                 asz,
@@ -558,6 +558,10 @@ func getHealReplicateObjectInfo(oi ObjectInfo, rcfg replicationConfig) Replicate
 		SSEC:                 crypto.SSEC.IsEncrypted(oi.UserDefined),
 		UserTags:             oi.UserTags,
 	}
+	if r.SSEC {
+		r.Checksum = oi.Checksum
+	}
+	return r
 }
 
 // ReplicationState - returns replication state using other internal replication metadata in ObjectInfo

cmd/bucket-replication.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -74,8 +75,9 @@ const (
 	ObjectLockRetentionTimestamp = "objectlock-retention-timestamp"
 	// ObjectLockLegalHoldTimestamp - the last time a legal hold metadata modification happened on this cluster for this object version
 	ObjectLockLegalHoldTimestamp = "objectlock-legalhold-timestamp"
-	// ReplicationWorkerMultiplier is suggested worker multiplier if traffic exceeds replication worker capacity
-	ReplicationWorkerMultiplier = 1.5
+
+	// ReplicationSsecChecksumHeader - the encrypted checksum of the SSE-C encrypted object.
+	ReplicationSsecChecksumHeader = "X-Minio-Replication-Ssec-Crc"
 )
 
 // gets replication config associated to a given bucket name.
@@ -763,12 +765,34 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
-func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
+func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) map[string]string {
 	meta := make(map[string]string)
+	cs := oi.decryptChecksums(partNum, h)
+	for k, v := range cs {
+		cksum := hash.NewChecksumString(k, v)
+		if cksum == nil {
+			continue
+		}
+		if cksum.Valid() {
+			meta[cksum.Type.Key()] = v
+		}
+	}
+	return meta
+}
+
+func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
+	meta := make(map[string]string)
+	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
+
 	for k, v := range objInfo.UserDefined {
 		// In case of SSE-C objects copy the allowed internal headers as well
-		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
+		if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
 			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+				if strings.EqualFold(k, ReservedMetadataPrefixLower+"crc") {
+					for k, v := range getCRCMeta(objInfo, partNum, nil) {
+						meta[k] = v
+					}
+				}
 				continue
 			}
 			if isStandardHeader(k) {
@@ -782,6 +806,17 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
 		}
 	}
 
+	if len(objInfo.Checksum) > 0 {
+		// Add encrypted CRC to metadata for SSE-C objects.
+		if isSSEC {
+			meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
+		} else {
+			for k, v := range getCRCMeta(objInfo, 0, nil) {
+				meta[k] = v
+			}
+		}
+	}
+
 	if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) {
 		sc = objInfo.StorageClass
 	}
@@ -1201,17 +1236,23 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 
-	size, err := objInfo.GetActualSize()
-	if err != nil {
-		replLogIf(ctx, err)
-		sendEvent(eventArgs{
-			EventName:  event.ObjectReplicationNotTracked,
-			BucketName: bucket,
-			Object:     objInfo,
-			UserAgent:  "Internal: [Replication]",
-			Host:       globalLocalNodeName,
-		})
-		return
+	// Set the encrypted size for SSE-C objects
+	var size int64
+	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+		size = objInfo.Size
+	} else {
+		size, err = objInfo.GetActualSize()
+		if err != nil {
+			replLogIf(ctx, err)
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
+			})
+			return
+		}
 	}
 
 	if tgt.Bucket == "" {
@@ -1239,7 +1280,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	// use core client to avoid doing multipart on PUT
 	c := &minio.Core{Client: tgt.Client}
 
-	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
+	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
 	if err != nil {
 		replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
 		sendEvent(eventArgs{
@@ -1377,7 +1418,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	// Set the encrypted size for SSE-C objects
-	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
+	if isSSEC {
 		size = objInfo.Size
 	}
 
@@ -1437,6 +1479,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			return
 		}
 	} else {
+		// SSEC objects will refuse HeadObject without the decryption key.
+		// Ignore the error, since we know the object exists and versioning prevents overwriting existing versions.
+		if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) {
+			rinfo.ReplicationStatus = replication.Completed
+			rinfo.ReplicationAction = replicateNone
+			goto applyAction
+		}
 		// if target returns error other than NoSuchKey, defer replication attempt
 		if minio.IsNetworkOrHostDown(cerr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
@@ -1464,6 +1513,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			return
 		}
 	}
+applyAction:
 	rinfo.ReplicationStatus = replication.Completed
 	rinfo.Size = size
 	rinfo.ReplicationAction = rAction
@@ -1506,7 +1556,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
-		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
+		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
 		if err != nil {
 			replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
 			sendEvent(eventArgs{
@@ -1597,13 +1647,14 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	}()
 
 	var (
-		hr    *hash.Reader
-		pInfo minio.ObjectPart
+		hr     *hash.Reader
+		pInfo  minio.ObjectPart
+		isSSEC = crypto.SSEC.IsEncrypted(objInfo.UserDefined)
 	)
 
 	var objectSize int64
 	for _, partInfo := range objInfo.Parts {
-		if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+		if isSSEC {
 			hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize)
 		} else {
 			hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
@@ -1614,12 +1665,18 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 
 		cHeader := http.Header{}
 		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
+		if !isSSEC {
+			crc := getCRCMeta(objInfo, partInfo.Number, nil) // No SSE-C keys here.
+			for k, v := range crc {
+				cHeader.Add(k, v)
+			}
+		}
 		popts := minio.PutObjectPartOptions{
 			SSE:          opts.ServerSideEncryption,
 			CustomHeader: cHeader,
 		}
 
-		if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+		if isSSEC {
 			objectSize += partInfo.Size
 			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts)
 		} else {
@@ -1629,20 +1686,30 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err != nil {
 			return err
 		}
-		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) && pInfo.Size != partInfo.ActualSize {
+		if !isSSEC && pInfo.Size != partInfo.ActualSize {
 			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
 		}
 		uploadedParts = append(uploadedParts, minio.CompletePart{
-			PartNumber: pInfo.PartNumber,
-			ETag:       pInfo.ETag,
+			PartNumber:     pInfo.PartNumber,
+			ETag:           pInfo.ETag,
+			ChecksumCRC32:  pInfo.ChecksumCRC32,
+			ChecksumCRC32C: pInfo.ChecksumCRC32C,
+			ChecksumSHA1:   pInfo.ChecksumSHA1,
+			ChecksumSHA256: pInfo.ChecksumSHA256,
 		})
 	}
+	userMeta := map[string]string{
+		validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"],
+	}
+	if isSSEC && objInfo.UserDefined[ReplicationSsecChecksumHeader] != "" {
+		userMeta[ReplicationSsecChecksumHeader] = objInfo.UserDefined[ReplicationSsecChecksumHeader]
+	}
 
 	// really big value but its okay on heavily loaded systems. This is just tail end timeout.
 	cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer ccancel()
 	_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
-		UserMetadata: map[string]string{validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"]},
+		UserMetadata: userMeta,
 		Internal: minio.AdvancedPutOptions{
 			SourceMTime: objInfo.ModTime,
 			SourceETag:  objInfo.ETag,
@@ -2179,12 +2246,12 @@ type proxyResult struct {
 
 // get Reader from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, _ http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) {
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) {
 	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
 	if !proxy.Proxy {
 		return nil, proxy, nil
 	}
-	fn, _, _, err := NewGetObjectReader(nil, oi, opts)
+	fn, _, _, err := NewGetObjectReader(nil, oi, opts, h)
 	if err != nil {
 		return nil, proxy, err
 	}
@@ -2369,7 +2436,9 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
 		SSEC:                 crypto.SSEC.IsEncrypted(oi.UserDefined),
 		UserTags:             oi.UserTags,
 	}
-
+	if ri.SSEC {
+		ri.Checksum = oi.Checksum
+	}
 	if dsc.Synchronous() {
 		replicateObject(ctx, ri, o)
 	} else {
@@ -2849,17 +2918,19 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 						ReplicationProxyRequest: "false",
 					},
 				})
+				sz := roi.Size
 				if err != nil {
 					if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
 						st.ReplicatedCount++
 					} else {
 						st.FailedCount++
 					}
+					sz = 0
 				} else {
 					st.ReplicatedCount++
 					st.ReplicatedSize += roi.Size
 				}
-				traceFn(err)
+				traceFn(sz, err)
 				select {
 				case <-ctx.Done():
 					return
@@ -2974,17 +3045,17 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 	return nil
 }
 
-func (s *replicationResyncer) trace(resyncID string, path string) func(err error) {
+func (s *replicationResyncer) trace(resyncID string, path string) func(sz int64, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(sz int64, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceReplicationResync) > 0 {
-			globalTrace.Publish(replicationResyncTrace(resyncID, startTime, duration, path, err))
+			globalTrace.Publish(replicationResyncTrace(resyncID, startTime, duration, path, err, sz))
 		}
 	}
 }
 
-func replicationResyncTrace(resyncID string, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
+func replicationResyncTrace(resyncID string, startTime time.Time, duration time.Duration, path string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -2998,6 +3069,7 @@ func replicationResyncTrace(resyncID string, startTime time.Time, duration time.
 		Duration:  duration,
 		Path:      path,
 		Error:     errStr,
+		Bytes:     sz,
 	}
 }
 

cmd/common-main.go

@@ -402,6 +402,29 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 		ctxt.certsDirSet = true
 	}
 
+	memAvailable := availableMemory()
+	if ctx.IsSet("memlimit") || ctx.GlobalIsSet("memlimit") {
+		memlimit := ctx.String("memlimit")
+		if memlimit == "" {
+			memlimit = ctx.GlobalString("memlimit")
+		}
+		mlimit, err := humanize.ParseBytes(memlimit)
+		if err != nil {
+			return err
+		}
+		if mlimit > memAvailable {
+			logger.Info("WARNING: maximum memory available (%s) smaller than specified --memlimit=%s, ignoring --memlimit value",
+				humanize.IBytes(memAvailable), memlimit)
+		}
+		ctxt.MemLimit = mlimit
+	} else {
+		ctxt.MemLimit = memAvailable
+	}
+
+	if memAvailable < ctxt.MemLimit {
+		ctxt.MemLimit = memAvailable
+	}
+
 	ctxt.FTP = ctx.StringSlice("ftp")
 	ctxt.SFTP = ctx.StringSlice("sftp")
 	ctxt.Interface = ctx.String("interface")

cmd/config-current.go

@@ -240,6 +240,11 @@ func initHelp() {
 			Description: "manage Browser HTTP specific features, such as Security headers, etc.",
 			Optional:    true,
 		},
+		config.HelpKV{
+			Key:         config.ILMSubSys,
+			Description: "manage ILM settings for expiration and transition workers",
+			Optional:    true,
+		},
 	}
 
 	if globalIsErasure {
@@ -288,6 +293,7 @@ func initHelp() {
 		config.DriveSubSys:          drive.HelpDrive,
 		config.CacheSubSys:          cache.Help,
 		config.BrowserSubSys:        browser.Help,
+		config.ILMSubSys:            ilm.Help,
 	}
 
 	config.RegisterHelpSubSys(helpMap)

cmd/data-scanner.go

@@ -1233,7 +1233,7 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 }
 
 func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool {
-	if obj.DeleteMarker {
+	if obj.DeleteMarker || obj.IsDir {
 		return false
 	}
 	globalTransitionState.queueTransitionTask(obj, event, src)

cmd/encryption-v1.go

@@ -1025,7 +1025,9 @@ func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err e
 	if encrypted {
 		if crypto.SSEC.IsEncrypted(info.UserDefined) {
 			if !(crypto.SSEC.IsRequested(headers) || crypto.SSECopy.IsRequested(headers)) {
-				return encrypted, errEncryptedObject
+				if r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" {
+					return encrypted, errEncryptedObject
+				}
 			}
 		}
 
@@ -1077,13 +1079,16 @@ func metadataEncrypter(key crypto.ObjectKey) objectMetaEncryptFn {
 }
 
 // metadataDecrypter reverses metadataEncrypter.
-func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
+func (o *ObjectInfo) metadataDecrypter(h http.Header) objectMetaDecryptFn {
 	return func(baseKey string, input []byte) ([]byte, error) {
 		if len(input) == 0 {
 			return input, nil
 		}
-
-		key, err := decryptObjectMeta(nil, o.Bucket, o.Name, o.UserDefined)
+		var key []byte
+		if k, err := crypto.SSEC.ParseHTTP(h); err == nil {
+			key = k[:]
+		}
+		key, err := decryptObjectMeta(key, o.Bucket, o.Name, o.UserDefined)
 		if err != nil {
 			return nil, err
 		}
@@ -1095,13 +1100,13 @@ func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
 
 // decryptChecksums will attempt to decode checksums and return it/them if set.
 // if part > 0, and we have the checksum for the part that will be returned.
-func (o *ObjectInfo) decryptPartsChecksums() {
+func (o *ObjectInfo) decryptPartsChecksums(h http.Header) {
 	data := o.Checksum
 	if len(data) == 0 {
 		return
 	}
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
-		decrypted, err := o.metadataDecrypter()("object-checksum", data)
+		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {
 			encLogIf(GlobalContext, err)
 			return
@@ -1157,15 +1162,17 @@ func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn
 
 // decryptChecksums will attempt to decode checksums and return it/them if set.
 // if part > 0, and we have the checksum for the part that will be returned.
-func (o *ObjectInfo) decryptChecksums(part int) map[string]string {
+func (o *ObjectInfo) decryptChecksums(part int, h http.Header) map[string]string {
 	data := o.Checksum
 	if len(data) == 0 {
 		return nil
 	}
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
-		decrypted, err := o.metadataDecrypter()("object-checksum", data)
+		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {
-			encLogIf(GlobalContext, err)
+			if err != crypto.ErrSecretKeyMismatch {
+				encLogIf(GlobalContext, err)
+			}
 			return nil
 		}
 		data = decrypted

cmd/erasure-healing-common.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"slices"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
@@ -253,6 +254,33 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error,
 	return onlineDisks, modTime, ""
 }
 
+// Convert verify or check parts returned error to integer representation
+func convPartErrToInt(err error) int {
+	err = unwrapAll(err)
+	switch err {
+	case nil:
+		return checkPartSuccess
+	case errFileNotFound, errFileVersionNotFound:
+		return checkPartFileNotFound
+	case errFileCorrupt:
+		return checkPartFileCorrupt
+	case errVolumeNotFound:
+		return checkPartVolumeNotFound
+	case errDiskNotFound:
+		return checkPartDiskNotFound
+	default:
+		return checkPartUnknown
+	}
+}
+
+func partNeedsHealing(partErrs []int) bool {
+	return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess && i != checkPartUnknown }) > -1
+}
+
+func hasPartErr(partErrs []int) bool {
+	return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess }) > -1
+}
+
 // disksWithAllParts - This function needs to be called with
 // []StorageAPI returned by listOnlineDisks. Returns,
 //
@@ -262,10 +290,19 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error,
 //     a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
 	errs []error, latestMeta FileInfo, bucket, object string,
-	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time,
-) {
-	availableDisks := make([]StorageAPI, len(onlineDisks))
-	dataErrs := make([]error, len(onlineDisks))
+	scanMode madmin.HealScanMode,
+) (availableDisks []StorageAPI, dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) {
+	availableDisks = make([]StorageAPI, len(onlineDisks))
+
+	dataErrsByDisk = make(map[int][]int, len(onlineDisks))
+	for i := range onlineDisks {
+		dataErrsByDisk[i] = make([]int, len(latestMeta.Parts))
+	}
+
+	dataErrsByPart = make(map[int][]int, len(latestMeta.Parts))
+	for i := range latestMeta.Parts {
+		dataErrsByPart[i] = make([]int, len(onlineDisks))
+	}
 
 	inconsistent := 0
 	for i, meta := range partsMetadata {
@@ -295,19 +332,21 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		erasureDistributionReliable = false
 	}
 
+	metaErrs := make([]error, len(errs))
+
 	for i, onlineDisk := range onlineDisks {
 		if errs[i] != nil {
-			dataErrs[i] = errs[i]
+			metaErrs[i] = errs[i]
 			continue
 		}
 		if onlineDisk == OfflineDisk {
-			dataErrs[i] = errDiskNotFound
+			metaErrs[i] = errDiskNotFound
 			continue
 		}
 
 		meta := partsMetadata[i]
 		if !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir {
-			dataErrs[i] = errFileCorrupt
+			metaErrs[i] = errFileCorrupt
 			partsMetadata[i] = FileInfo{}
 			continue
 		}
@@ -315,7 +354,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		if erasureDistributionReliable {
 			if !meta.IsValid() {
 				partsMetadata[i] = FileInfo{}
-				dataErrs[i] = errFileCorrupt
+				metaErrs[i] = errFileCorrupt
 				continue
 			}
 
@@ -325,46 +364,79 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 					// attempt a fix if possible, assuming other entries
 					// might have the right erasure distribution.
 					partsMetadata[i] = FileInfo{}
-					dataErrs[i] = errFileCorrupt
+					metaErrs[i] = errFileCorrupt
 					continue
 				}
 			}
 		}
+	}
+
+	// Copy meta errors to part errors
+	for i, err := range metaErrs {
+		if err != nil {
+			partErr := convPartErrToInt(err)
+			for p := range latestMeta.Parts {
+				dataErrsByPart[p][i] = partErr
+			}
+		}
+	}
+
+	for i, onlineDisk := range onlineDisks {
+		if metaErrs[i] != nil {
+			continue
+		}
+		meta := partsMetadata[i]
+
+		if meta.Deleted || meta.IsRemote() {
+			continue
+		}
 
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
-			dataErrs[i] = bitrotVerify(bytes.NewReader(meta.Data),
+			verifyErr := bitrotVerify(bytes.NewReader(meta.Data),
 				int64(len(meta.Data)),
 				meta.Erasure.ShardFileSize(meta.Size),
 				checksumInfo.Algorithm,
 				checksumInfo.Hash, meta.Erasure.ShardSize())
-			if dataErrs[i] == nil {
-				// All parts verified, mark it as all data available.
-				availableDisks[i] = onlineDisk
-			} else {
-				// upon errors just make that disk's fileinfo invalid
-				partsMetadata[i] = FileInfo{}
-			}
+			dataErrsByPart[0][i] = convPartErrToInt(verifyErr)
 			continue
 		}
 
+		var (
+			verifyErr  error
+			verifyResp *CheckPartsResp
+		)
+
 		meta.DataDir = latestMeta.DataDir
 		switch scanMode {
 		case madmin.HealDeepScan:
 			// disk has a valid xl.meta but may not have all the
 			// parts. This is considered an outdated disk, since
 			// it needs healing too.
-			if !meta.Deleted && !meta.IsRemote() {
-				dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, meta)
-			}
-		case madmin.HealNormalScan:
-			if !meta.Deleted && !meta.IsRemote() {
-				dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, meta)
-			}
+			verifyResp, verifyErr = onlineDisk.VerifyFile(ctx, bucket, object, meta)
+		default:
+			verifyResp, verifyErr = onlineDisk.CheckParts(ctx, bucket, object, meta)
 		}
 
-		if dataErrs[i] == nil {
+		for p := range latestMeta.Parts {
+			if verifyErr != nil {
+				dataErrsByPart[p][i] = convPartErrToInt(verifyErr)
+			} else {
+				dataErrsByPart[p][i] = verifyResp.Results[p]
+			}
+		}
+	}
+
+	// Build dataErrs by disk from dataErrs by part
+	for part, disks := range dataErrsByPart {
+		for disk := range disks {
+			dataErrsByDisk[disk][part] = dataErrsByPart[part][disk]
+		}
+	}
+
+	for i, onlineDisk := range onlineDisks {
+		if metaErrs[i] == nil && !hasPartErr(dataErrsByDisk[i]) {
 			// All parts verified, mark it as all data available.
 			availableDisks[i] = onlineDisk
 		} else {
@@ -373,5 +445,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	return availableDisks, dataErrs, timeSentinel
+	return
 }

cmd/erasure-healing-common_test.go

@@ -308,9 +308,8 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
 			}
-			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+			availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 				test.errs, fi, bucket, object, madmin.HealDeepScan)
-			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
@@ -491,9 +490,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					test.expectedTime, modTime)
 			}
 
-			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+			availableDisks, _, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 				test.errs, fi, bucket, object, madmin.HealDeepScan)
-			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
@@ -554,7 +552,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)
 
-	filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
+	filteredDisks, _, dataErrsPerDisk := disksWithAllParts(ctx, erasureDisks, partsMetadata,
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
@@ -562,8 +560,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	for diskIndex, disk := range filteredDisks {
-		if errs[diskIndex] != nil {
-			t.Errorf("Unexpected error %s", errs[diskIndex])
+		if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
+			t.Errorf("Unexpected error: %v", dataErrsPerDisk[diskIndex])
 		}
 
 		if disk == nil {
@@ -634,7 +632,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, errs, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+	filteredDisks, dataErrsPerDisk, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
@@ -646,15 +644,15 @@ func TestDisksWithAllParts(t *testing.T) {
 			if disk != nil {
 				t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 			}
-			if errs[diskIndex] == nil {
-				t.Errorf("Expected error not received, driveIndex: %d", diskIndex)
+			if !partNeedsHealing(dataErrsPerDisk[diskIndex]) {
+				t.Errorf("Disk expected to be healed, driveIndex: %d", diskIndex)
 			}
 		} else {
 			if disk == nil {
 				t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 			}
-			if errs[diskIndex] != nil {
-				t.Errorf("Unexpected error, %s, driveIndex: %d", errs[diskIndex], diskIndex)
+			if partNeedsHealing(dataErrsPerDisk[diskIndex]) {
+				t.Errorf("Disk not expected to be healed, driveIndex: %d", diskIndex)
 			}
 
 		}

cmd/erasure-healing.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/sync/errgroup"
+	"golang.org/x/exp/slices"
 )
 
 //go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
@@ -144,36 +145,41 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 	return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum)
 }
 
+var errLegacyXLMeta = errors.New("legacy XL meta")
+
+var errOutdatedXLMeta = errors.New("outdated XL meta")
+
+var errPartMissingOrCorrupt = errors.New("part missing or corrupt")
+
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
-	switch {
-	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
-		return true
-	case errors.Is(erErr, errFileCorrupt):
-		return true
+func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, error) {
+	if errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound) || errors.Is(erErr, errFileCorrupt) {
+		return true, erErr
 	}
 	if erErr == nil {
 		if meta.XLV1 {
 			// Legacy means heal always
 			// always check first.
-			return true
+			return true, errLegacyXLMeta
+		}
+		if !latestMeta.Equals(meta) {
+			return true, errOutdatedXLMeta
 		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
-			if IsErr(dataErr, []error{
-				errFileNotFound,
-				errFileVersionNotFound,
-				errFileCorrupt,
-			}...) {
-				return true
+			for _, partErr := range partsErrs {
+				if slices.Contains([]int{
+					checkPartFileNotFound,
+					checkPartFileCorrupt,
+				}, partErr) {
+					return true, errPartMissingOrCorrupt
+				}
 			}
 		}
-		if !latestMeta.Equals(meta) {
-			return true
-		}
+		return false, nil
 	}
-	return false
+	return false, erErr
 }
 
 const (
@@ -332,7 +338,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+	availableDisks, dataErrsByDisk, dataErrsByPart := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
 	var erasure Erasure
@@ -355,15 +361,20 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	// to be healed.
 	outDatedDisks := make([]StorageAPI, len(storageDisks))
 	disksToHealCount := 0
-	for i, v := range availableDisks {
+	for i := range availableDisks {
+		yes, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta)
+		if yes {
+			outDatedDisks[i] = storageDisks[i]
+			disksToHealCount++
+		}
+
 		driveState := ""
 		switch {
-		case v != nil:
+		case reason == nil:
 			driveState = madmin.DriveStateOk
-		case errors.Is(errs[i], errDiskNotFound), errors.Is(dataErrs[i], errDiskNotFound):
+		case IsErr(reason, errDiskNotFound):
 			driveState = madmin.DriveStateOffline
-		case IsErr(errs[i], errFileNotFound, errFileVersionNotFound, errVolumeNotFound),
-			IsErr(dataErrs[i], errFileNotFound, errFileVersionNotFound, errVolumeNotFound):
+		case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissingOrCorrupt, errOutdatedXLMeta, errLegacyXLMeta):
 			driveState = madmin.DriveStateMissing
 		default:
 			// all remaining cases imply corrupt data/metadata
@@ -380,12 +391,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			Endpoint: storageEndpoints[i].String(),
 			State:    driveState,
 		})
-
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
-			outDatedDisks[i] = storageDisks[i]
-			disksToHealCount++
-			continue
-		}
 	}
 
 	if isAllNotFound(errs) {
@@ -412,7 +417,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// Allow for dangling deletes, on versions that have DataDir missing etc.
 		// this would end up restoring the correct readable versions.
-		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{
+		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrsByPart, ObjectOptions{
 			VersionID: versionID,
 		})
 		errs = make([]error, len(errs))
@@ -908,35 +913,52 @@ func isObjectDirDangling(errs []error) (ok bool) {
 	return found < notFound && found > 0
 }
 
+func danglingMetaErrsCount(cerrs []error) (notFoundCount int, nonActionableCount int) {
+	for _, readErr := range cerrs {
+		if readErr == nil {
+			continue
+		}
+		switch {
+		case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
+			notFoundCount++
+		default:
+			// All other errors are non-actionable
+			nonActionableCount++
+		}
+	}
+	return
+}
+
+func danglingPartErrsCount(results []int) (notFoundCount int, nonActionableCount int) {
+	for _, partResult := range results {
+		switch partResult {
+		case checkPartSuccess:
+			continue
+		case checkPartFileNotFound:
+			notFoundCount++
+		default:
+			// All other errors are non-actionable
+			nonActionableCount++
+		}
+	}
+	return
+}
+
 // Object is considered dangling/corrupted if and only
 // if total disks - a combination of corrupted and missing
 // files is lesser than number of data blocks.
-func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) {
+func isObjectDangling(metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int) (validMeta FileInfo, ok bool) {
 	// We can consider an object data not reliable
 	// when xl.meta is not found in read quorum disks.
 	// or when xl.meta is not readable in read quorum disks.
-	danglingErrsCount := func(cerrs []error) (int, int) {
-		var (
-			notFoundCount      int
-			nonActionableCount int
-		)
-		for _, readErr := range cerrs {
-			if readErr == nil {
-				continue
-			}
-			switch {
-			case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
-				notFoundCount++
-			default:
-				// All other errors are non-actionable
-				nonActionableCount++
-			}
-		}
-		return notFoundCount, nonActionableCount
-	}
+	notFoundMetaErrs, nonActionableMetaErrs := danglingMetaErrsCount(errs)
 
-	notFoundMetaErrs, nonActionableMetaErrs := danglingErrsCount(errs)
-	notFoundPartsErrs, nonActionablePartsErrs := danglingErrsCount(dataErrs)
+	notFoundPartsErrs, nonActionablePartsErrs := 0, 0
+	for _, dataErrs := range dataErrsByPart {
+		if nf, na := danglingPartErrsCount(dataErrs); nf > notFoundPartsErrs {
+			notFoundPartsErrs, nonActionablePartsErrs = nf, na
+		}
+	}
 
 	for _, m := range metaArr {
 		if m.IsValid() {
@@ -948,7 +970,7 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	if !validMeta.IsValid() {
 		// validMeta is invalid because all xl.meta is missing apparently
 		// we should figure out if dataDirs are also missing > dataBlocks.
-		dataBlocks := (len(dataErrs) + 1) / 2
+		dataBlocks := (len(metaArr) + 1) / 2
 		if notFoundPartsErrs > dataBlocks {
 			// Not using parity to ensure that we do not delete
 			// any valid content, if any is recoverable. But if
@@ -1065,6 +1087,7 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object strin
 		if result != nil {
 			tr.Custom["version-id"] = result.VersionID
 			tr.Custom["disks"] = strconv.Itoa(result.DiskCount)
+			tr.Bytes = result.ObjectSize
 		}
 	}
 	if err != nil {

cmd/erasure-healing_test.go

@@ -49,7 +49,7 @@ func TestIsObjectDangling(t *testing.T) {
 		name             string
 		metaArr          []FileInfo
 		errs             []error
-		dataErrs         []error
+		dataErrs         map[int][]int
 		expectedMeta     FileInfo
 		expectedDangling bool
 	}{
@@ -165,11 +165,8 @@ func TestIsObjectDangling(t *testing.T) {
 				nil,
 				nil,
 			},
-			dataErrs: []error{
-				errFileCorrupt,
-				errFileNotFound,
-				nil,
-				errFileCorrupt,
+			dataErrs: map[int][]int{
+				0: {checkPartFileCorrupt, checkPartFileNotFound, checkPartSuccess, checkPartFileCorrupt},
 			},
 			expectedMeta:     fi,
 			expectedDangling: false,
@@ -188,11 +185,8 @@ func TestIsObjectDangling(t *testing.T) {
 				errFileNotFound,
 				nil,
 			},
-			dataErrs: []error{
-				errFileNotFound,
-				errFileCorrupt,
-				nil,
-				nil,
+			dataErrs: map[int][]int{
+				0: {checkPartFileNotFound, checkPartFileCorrupt, checkPartSuccess, checkPartSuccess},
 			},
 			expectedMeta:     fi,
 			expectedDangling: false,
@@ -247,15 +241,58 @@ func TestIsObjectDangling(t *testing.T) {
 				nil,
 				nil,
 			},
-			dataErrs: []error{
-				errFileNotFound,
-				errFileNotFound,
-				nil,
-				errFileNotFound,
+			dataErrs: map[int][]int{
+				0: {checkPartFileNotFound, checkPartFileNotFound, checkPartSuccess, checkPartFileNotFound},
 			},
 			expectedMeta:     fi,
 			expectedDangling: true,
 		},
+		{
+			name: "FileInfoDecided-case4-(missing data-dir for part 2)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+				nil,
+			},
+			dataErrs: map[int][]int{
+				0: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartSuccess},
+				1: {checkPartSuccess, checkPartFileNotFound, checkPartFileNotFound, checkPartFileNotFound},
+			},
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+
+		{
+			name: "FileInfoDecided-case4-(enough data-dir existing for each part)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+				nil,
+			},
+			dataErrs: map[int][]int{
+				0: {checkPartFileNotFound, checkPartSuccess, checkPartSuccess, checkPartSuccess},
+				1: {checkPartSuccess, checkPartFileNotFound, checkPartSuccess, checkPartSuccess},
+				2: {checkPartSuccess, checkPartSuccess, checkPartFileNotFound, checkPartSuccess},
+				3: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartFileNotFound},
+			},
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+
 		// Add new cases as seen
 	}
 	for _, testCase := range testCases {

cmd/erasure-multipart.go

@@ -483,6 +483,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	}
 	fi.DataDir = mustGetUUID()
 
+	if userDefined[ReplicationSsecChecksumHeader] != "" {
+		fi.Checksum, _ = base64.StdEncoding.DecodeString(userDefined[ReplicationSsecChecksumHeader])
+		delete(userDefined, ReplicationSsecChecksumHeader)
+	}
+
 	// Initialize erasure metadata.
 	for index := range partsMetadata {
 		partsMetadata[index] = fi
@@ -1294,9 +1299,18 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		defer lk.Unlock(lkctx)
 	}
 
+	// Accept encrypted checksum from incoming request.
+	if opts.UserDefined[ReplicationSsecChecksumHeader] != "" {
+		if v, err := base64.StdEncoding.DecodeString(opts.UserDefined[ReplicationSsecChecksumHeader]); err == nil {
+			fi.Checksum = v
+		}
+		delete(opts.UserDefined, ReplicationSsecChecksumHeader)
+	}
+
 	if checksumType.IsSet() {
 		checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
-		cs := hash.NewChecksumFromData(checksumType, checksumCombined)
+		var cs *hash.Checksum
+		cs = hash.NewChecksumFromData(checksumType, checksumCombined)
 		fi.Checksum = cs.AppendTo(nil, checksumCombined)
 		if opts.EncryptFn != nil {
 			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)

cmd/erasure-object.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -276,7 +277,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return gr.WithCleanupFuncs(nsUnlocker), nil
 	}
 
-	fn, off, length, err := NewGetObjectReader(rs, objInfo, opts)
+	fn, off, length, err := NewGetObjectReader(rs, objInfo, opts, h)
 	if err != nil {
 		return nil, err
 	}
@@ -483,8 +484,8 @@ func joinErrs(errs []error) []string {
 	return s
 }
 
-func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
-	m, ok := isObjectDangling(metaArr, errs, dataErrs)
+func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) {
+	m, ok := isObjectDangling(metaArr, errs, dataErrsByPart)
 	if !ok {
 		// We only come here if we cannot figure out if the object
 		// can be deleted safely, in such a scenario return ReadQuorum error.
@@ -494,7 +495,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 	tags["set"] = er.setIndex
 	tags["pool"] = er.poolIndex
 	tags["merrs"] = joinErrs(errs)
-	tags["derrs"] = joinErrs(dataErrs)
+	tags["derrs"] = dataErrsByPart
 	if m.IsValid() {
 		tags["size"] = m.Size
 		tags["mtime"] = m.ModTime.Format(http.TimeFormat)
@@ -508,8 +509,20 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 
 	// count the number of offline disks
 	offline := 0
-	for i := 0; i < max(len(errs), len(dataErrs)); i++ {
-		if i < len(errs) && errors.Is(errs[i], errDiskNotFound) || i < len(dataErrs) && errors.Is(dataErrs[i], errDiskNotFound) {
+	for i := 0; i < len(errs); i++ {
+		var found bool
+		switch {
+		case errors.Is(errs[i], errDiskNotFound):
+			found = true
+		default:
+			for p := range dataErrsByPart {
+				if dataErrsByPart[p][i] == checkPartDiskNotFound {
+					found = true
+					break
+				}
+			}
+		}
+		if found {
 			offline++
 		}
 	}
@@ -1355,6 +1368,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	if opts.EncryptFn != nil {
 		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
 	}
+	if userDefined[ReplicationSsecChecksumHeader] != "" {
+		if v, err := base64.StdEncoding.DecodeString(userDefined[ReplicationSsecChecksumHeader]); err == nil {
+			fi.Checksum = v
+		}
+	}
+	delete(userDefined, ReplicationSsecChecksumHeader)
 	uniqueID := mustGetUUID()
 	tempObj := uniqueID
 

cmd/erasure-server-pool-decom.go

@@ -836,7 +836,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				if filterLifecycle(bi.Name, version.Name, version) {
 					expired++
 					decommissioned++
-					stopFn(errors.New("ILM expired object/version will be skipped"))
+					stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
 					continue
 				}
 
@@ -846,7 +846,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				remainingVersions := len(fivs.Versions) - expired
 				if version.Deleted && remainingVersions == 1 {
 					decommissioned++
-					stopFn(errors.New("DELETE marked object with no other non-current versions will be skipped"))
+					stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
 					continue
 				}
 
@@ -877,7 +877,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							err = nil
 						}
 					}
-					stopFn(err)
+					stopFn(version.Size, err)
 					if err != nil {
 						decomLogIf(ctx, err)
 						failure = true
@@ -902,12 +902,12 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							MTime:       version.ModTime,
 							UserDefined: version.Metadata,
 						}); err != nil {
-							stopFn(err)
+							stopFn(version.Size, err)
 							failure = true
 							decomLogIf(ctx, err)
 							continue
 						}
-						stopFn(nil)
+						stopFn(version.Size, nil)
 						failure = false
 						break
 					}
@@ -925,14 +925,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
-						stopFn(nil)
+						stopFn(version.Size, nil)
 						break
 					}
 					if err != nil && !ignore {
 						// if usage-cache.bin is not readable log and ignore it.
 						if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) {
 							ignore = true
-							stopFn(err)
+							stopFn(version.Size, err)
 							decomLogIf(ctx, err)
 							break
 						}
@@ -940,16 +940,16 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if err != nil {
 						failure = true
 						decomLogIf(ctx, err)
-						stopFn(err)
+						stopFn(version.Size, err)
 						continue
 					}
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
-						stopFn(err)
+						stopFn(version.Size, err)
 						failure = true
 						decomLogIf(ctx, err)
 						continue
 					}
-					stopFn(nil)
+					stopFn(version.Size, nil)
 					failure = false
 					break
 				}
@@ -977,7 +977,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						NoAuditLog:         true,
 					},
 				)
-				stopFn(err)
+				stopFn(0, err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
 					decomLogIf(ctx, err)
@@ -1038,7 +1038,7 @@ const (
 	decomMetricDecommissionRemoveObject
 )
 
-func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
+func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -1051,15 +1051,16 @@ func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.D
 		Duration:  duration,
 		Path:      path,
 		Error:     errStr,
+		Bytes:     sz,
 	}
 }
 
-func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err error) {
+func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(z int64, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(sz int64, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceDecommission) > 0 {
-			globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err))
+			globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err, sz))
 		}
 	}
 }
@@ -1092,10 +1093,10 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		}
 		stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name)
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
-			stopFn(err)
+			stopFn(0, err)
 			return err
 		}
-		stopFn(nil)
+		stopFn(0, nil)
 
 		z.poolMetaMutex.Lock()
 		if z.poolMeta.BucketDone(idx, bucket) {
@@ -1326,11 +1327,7 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 	poolInfo := z.poolMeta.Pools[idx].Clone()
 	if poolInfo.Decommission != nil {
 		poolInfo.Decommission.TotalSize = pi.Total
-		if poolInfo.Decommission.Failed || poolInfo.Decommission.Canceled {
-			poolInfo.Decommission.CurrentSize = pi.Free
-		} else {
-			poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
-		}
+		poolInfo.Decommission.CurrentSize = pi.Free
 	} else {
 		poolInfo.Decommission = &PoolDecommissionInfo{
 			TotalSize:   pi.Total,

cmd/erasure-server-pool-rebalance.go

@@ -427,7 +427,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 
 			stopFn := globalRebalanceMetrics.log(rebalanceMetricSaveMetadata, poolIdx, traceMsg)
 			err := z.saveRebalanceStats(GlobalContext, poolIdx, rebalSaveStats)
-			stopFn(err)
+			stopFn(0, err)
 			rebalanceLogIf(GlobalContext, err)
 
 			if quit {
@@ -456,7 +456,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 
 		stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBucket, poolIdx, bucket)
 		if err = z.rebalanceBucket(ctx, bucket, poolIdx); err != nil {
-			stopFn(err)
+			stopFn(0, err)
 			if errors.Is(err, errServerNotInitialized) || errors.Is(err, errBucketMetadataNotInitialized) {
 				continue
 			}
@@ -464,7 +464,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 			doneCh <- err
 			return
 		}
-		stopFn(nil)
+		stopFn(0, nil)
 		z.bucketRebalanceDone(bucket, poolIdx)
 	}
 
@@ -692,24 +692,24 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
-						stopFn(nil)
+						stopFn(0, nil)
 						break
 					}
 					if err != nil {
 						failure = true
 						rebalanceLogIf(ctx, err)
-						stopFn(err)
+						stopFn(0, err)
 						continue
 					}
 
 					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
 						failure = true
 						rebalanceLogIf(ctx, err)
-						stopFn(err)
+						stopFn(version.Size, err)
 						continue
 					}
 
-					stopFn(nil)
+					stopFn(version.Size, nil)
 					failure = false
 					break
 				}
@@ -735,7 +735,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						NoAuditLog:         true,
 					},
 				)
-				stopFn(err)
+				stopFn(0, err)
 				auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
 				if err != nil {
 					rebalanceLogIf(ctx, err)
@@ -935,7 +935,7 @@ func (z *erasureServerPools) StartRebalance() {
 		go func(idx int) {
 			stopfn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBuckets, idx)
 			err := z.rebalanceBuckets(ctx, idx)
-			stopfn(err)
+			stopfn(0, err)
 		}(poolIdx)
 	}
 }
@@ -975,7 +975,7 @@ const (
 	rebalanceMetricSaveMetadata
 )
 
-func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string) madmin.TraceInfo {
+func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -988,15 +988,16 @@ func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duratio
 		Duration:  duration,
 		Path:      path,
 		Error:     errStr,
+		Bytes:     sz,
 	}
 }
 
-func (p *rebalanceMetrics) log(r rebalanceMetric, poolIdx int, paths ...string) func(err error) {
+func (p *rebalanceMetrics) log(r rebalanceMetric, poolIdx int, paths ...string) func(sz int64, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(sz int64, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceRebalance) > 0 {
-			globalTrace.Publish(rebalanceTrace(r, poolIdx, startTime, duration, err, strings.Join(paths, " ")))
+			globalTrace.Publish(rebalanceTrace(r, poolIdx, startTime, duration, err, strings.Join(paths, " "), sz))
 		}
 	}
 }

cmd/erasure-server-pool.go

@@ -104,7 +104,11 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
 	buffers := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
-	buffers.Populate()
+	if n >= 16384 {
+		// pre-populate buffers only n >= 16384 which is (32Gi/2Mi)
+		// for all setups smaller than this avoid pre-alloc.
+		buffers.Populate()
+	}
 	globalBytePoolCap.Store(buffers)
 
 	var localDrives []StorageAPI
@@ -165,6 +169,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	if !globalIsDistErasure {
 		globalLocalDrivesMu.Lock()
 		globalLocalDrives = localDrives
+		for _, drive := range localDrives {
+			globalLocalDrivesMap[drive.Endpoint().String()] = drive
+		}
 		globalLocalDrivesMu.Unlock()
 	}
 
@@ -1535,7 +1542,7 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 		return loi, nil
 	}
 	ri := logger.GetReqInfo(ctx)
-	hadoop := ri != nil && strings.Contains(ri.UserAgent, `Hadoop `) && strings.Contains(ri.UserAgent, "scala/")
+	hadoop := ri != nil && strings.Contains(ri.UserAgent, "Hadoop ") && strings.Contains(ri.UserAgent, "scala/")
 	matches := func() bool {
 		if prefix == "" {
 			return false
@@ -1600,7 +1607,7 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 		// }
 		if matches() {
 			objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true})
-			if err == nil {
+			if err == nil || objInfo.IsLatest && objInfo.DeleteMarker {
 				if opts.Lifecycle != nil {
 					evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
 					if evt.Action.Delete() {
@@ -1626,41 +1633,29 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 		// and throw a 404 exception. This is especially a problem for spark jobs overwriting the same partition
 		// repeatedly. This workaround recursively lists the top 3 entries including delete markers to reflect the
 		// correct state of the directory in the list results.
-		opts.Recursive = true
-		opts.InclDeleted = true
-		opts.Limit = maxKeys + 1
-		li, err := listFn(ctx, opts, opts.Limit)
-		if err == nil {
-			switch {
-			case len(li.Objects) == 0 && len(li.Prefixes) == 0:
-				return loi, nil
-			case len(li.Objects) > 0 || len(li.Prefixes) > 0:
-				var o ObjectInfo
-				var pfx string
-				if len(li.Objects) > 0 {
-					o = li.Objects[0]
-					p := strings.TrimPrefix(o.Name, opts.Prefix)
-					if p != "" {
-						sidx := strings.Index(p, "/")
-						if sidx > 0 {
-							pfx = p[:sidx]
-						}
-					}
-				}
-				switch {
-				case o.DeleteMarker:
-					loi.Objects = append(loi.Objects, ObjectInfo{Bucket: bucket, IsDir: true, Name: prefix})
-					return loi, nil
-				case len(li.Objects) >= 1:
-					loi.Objects = append(loi.Objects, o)
-					if pfx != "" {
-						loi.Prefixes = append(loi.Prefixes, path.Join(opts.Prefix, pfx))
-					}
-				case len(li.Prefixes) > 0:
-					loi.Prefixes = append(loi.Prefixes, li.Prefixes...)
-				}
+		if strings.HasSuffix(opts.Prefix, SlashSeparator) {
+			li, err := listFn(ctx, opts, maxKeys)
+			if err != nil {
+				return loi, err
 			}
-			return loi, nil
+			if len(li.Objects) == 0 {
+				prefixes := li.Prefixes[:0]
+				for _, prefix := range li.Prefixes {
+					objInfo, _ := z.GetObjectInfo(ctx, bucket, pathJoin(prefix, "_SUCCESS"), ObjectOptions{NoLock: true})
+					if objInfo.IsLatest && objInfo.DeleteMarker {
+						continue
+					}
+					prefixes = append(prefixes, prefix)
+				}
+				if len(prefixes) > 0 {
+					objInfo, _ := z.GetObjectInfo(ctx, bucket, pathJoin(opts.Prefix, "_SUCCESS"), ObjectOptions{NoLock: true})
+					if objInfo.IsLatest && objInfo.DeleteMarker {
+						return loi, nil
+					}
+				}
+				li.Prefixes = prefixes
+			}
+			return li, nil
 		}
 	}
 

cmd/ftp-server-driver.go

@@ -105,7 +105,7 @@ type ftpMetrics struct{}
 
 var globalFtpMetrics ftpMetrics
 
-func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error) madmin.TraceInfo {
+func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -114,25 +114,33 @@ func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err e
 		TraceType: madmin.TraceFTP,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
-		FuncName:  fmt.Sprintf("ftp USER=%s COMMAND=%s PARAM=%s ISLOGIN=%t, Source=%s", s.Sess.LoginUser(), s.Cmd, s.Param, s.Sess.IsLogin(), source),
+		FuncName:  fmt.Sprintf(s.Cmd),
 		Duration:  time.Since(startTime),
 		Path:      objPath,
 		Error:     errStr,
+		Bytes:     sz,
+		Custom: map[string]string{
+			"user":   s.Sess.LoginUser(),
+			"cmd":    s.Cmd,
+			"param":  s.Param,
+			"login":  fmt.Sprintf("%t", s.Sess.IsLogin()),
+			"source": source,
+		},
 	}
 }
 
-func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(err error) {
+func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(sz int64, err error) {
 	startTime := time.Now()
 	source := getSource(2)
-	return func(err error) {
-		globalTrace.Publish(ftpTrace(s, startTime, source, strings.Join(paths, " "), err))
+	return func(sz int64, err error) {
+		globalTrace.Publish(ftpTrace(s, startTime, source, strings.Join(paths, " "), err, sz))
 	}
 }
 
 // Stat implements ftpDriver
 func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo, err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	if objPath == SlashSeparator {
 		return &minioFileInfo{
@@ -190,7 +198,7 @@ func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo,
 // ListDir implements ftpDriver
 func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func(os.FileInfo) error) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	clnt, err := driver.getMinIOClient(ctx)
 	if err != nil {
@@ -252,7 +260,7 @@ func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func
 
 func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string) (ok bool, err error) {
 	stopFn := globalFtpMetrics.log(c, username)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	if globalIAMSys.LDAPConfig.Enabled() {
 		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), username)
@@ -376,7 +384,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 // DeleteDir implements ftpDriver
 func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
@@ -426,7 +434,7 @@ func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error)
 // DeleteFile implements ftpDriver
 func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
@@ -444,7 +452,7 @@ func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error
 // Rename implements ftpDriver
 func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, fromObjPath, toObjPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	return NotImplemented{}
 }
@@ -452,7 +460,7 @@ func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath
 // MakeDir implements ftpDriver
 func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
@@ -479,7 +487,7 @@ func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) {
 // GetFile implements ftpDriver
 func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64) (n int64, rc io.ReadCloser, err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(n, err)
 
 	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
@@ -511,14 +519,14 @@ func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64)
 	if err != nil {
 		return 0, nil, err
 	}
-
-	return info.Size - offset, obj, nil
+	n = info.Size - offset
+	return n, obj, nil
 }
 
 // PutFile implements ftpDriver
 func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reader, offset int64) (n int64, err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(n, err)
 
 	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
@@ -539,5 +547,6 @@ func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reade
 		ContentType:          mimedb.TypeByExtension(path.Ext(object)),
 		DisableContentSha256: true,
 	})
-	return info.Size, err
+	n = info.Size
+	return n, err
 }

cmd/generic-handlers.go

@@ -601,13 +601,6 @@ func setUploadForwardingMiddleware(h http.Handler) http.Handler {
 				h.ServeHTTP(w, r)
 				return
 			}
-			// forward request to peer handling this upload
-			if globalBucketTargetSys.isOffline(remote.EndpointURL) {
-				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
-				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrReplicationRemoteConnectionError), r.URL)
-				return
-			}
-
 			r.URL.Scheme = remote.EndpointURL.Scheme
 			r.URL.Host = remote.EndpointURL.Host
 			// Make sure we remove any existing headers before

cmd/global-heal.go

@@ -530,7 +530,16 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	tracker.setObject("")
 	tracker.setBucket("")
 
-	return retErr
+	if retErr != nil {
+		return retErr
+	}
+
+	// Last sanity check
+	if len(tracker.QueuedBuckets) > 0 {
+		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
+	}
+
+	return nil
 }
 
 func healBucket(bucket string, scan madmin.HealScanMode) error {

cmd/globals.go

@@ -160,6 +160,8 @@ type serverCtxt struct {
 	FTP  []string
 	SFTP []string
 
+	MemLimit uint64
+
 	UserTimeout         time.Duration
 	ShutdownTimeout     time.Duration
 	IdleTimeout         time.Duration

cmd/handler-api.go

@@ -91,11 +91,11 @@ func availableMemory() (available uint64) {
 	available = 2048 * blockSizeV2 * 2 // Default to 4 GiB when we can't find the limits.
 
 	if runtime.GOOS == "linux" {
-		// Useful in container mode
+		// Honor cgroup limits if set.
 		limit := cgroupMemLimit()
 		if limit > 0 {
-			// A valid value is found, return its 75%
-			available = (limit * 3) / 4
+			// A valid value is found, return its 90%
+			available = (limit * 9) / 10
 			return
 		}
 	} // for all other platforms limits are based on virtual memory.
@@ -104,8 +104,9 @@ func availableMemory() (available uint64) {
 	if err != nil {
 		return
 	}
-	// A valid value is available return its 75%
-	available = (memStats.Available * 3) / 4
+
+	// A valid value is available return its 90%
+	available = (memStats.Available * 9) / 10
 	return
 }
 
@@ -129,7 +130,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 		maxSetDrives := slices.Max(setDriveCounts)
 
 		// Returns 75% of max memory allowed
-		maxMem := availableMemory()
+		maxMem := globalServerCtxt.MemLimit
 
 		// max requests per node is calculated as
 		// total_ram / ram_per_request
@@ -322,10 +323,9 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		}
 
 		globalHTTPStats.addRequestsInQueue(1)
-		defer globalHTTPStats.addRequestsInQueue(-1)
-
 		pool, deadline := globalAPIConfig.getRequestsPool()
 		if pool == nil {
+			globalHTTPStats.addRequestsInQueue(-1)
 			f.ServeHTTP(w, r)
 			return
 		}
@@ -333,6 +333,7 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		// No deadline to wait, there is nothing to queue
 		// perform the API call immediately.
 		if deadline <= 0 {
+			globalHTTPStats.addRequestsInQueue(-1)
 			f.ServeHTTP(w, r)
 			return
 		}
@@ -348,12 +349,14 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		select {
 		case pool <- struct{}{}:
 			defer func() { <-pool }()
+			globalHTTPStats.addRequestsInQueue(-1)
 			if contextCanceled(ctx) {
 				w.WriteHeader(499)
 				return
 			}
 			f.ServeHTTP(w, r)
 		case <-deadlineTimer.C:
+			globalHTTPStats.addRequestsInQueue(-1)
 			if contextCanceled(ctx) {
 				w.WriteHeader(499)
 				return
@@ -363,6 +366,7 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 				errorCodes.ToAPIErr(ErrTooManyRequests),
 				r.URL)
 		case <-r.Context().Done():
+			globalHTTPStats.addRequestsInQueue(-1)
 			// When the client disconnects before getting the S3 handler
 			// status code response, set the status code to 499 so this request
 			// will be properly audited and traced.

cmd/handler-utils.go

@@ -33,8 +33,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 	xnet "github.com/minio/pkg/v3/net"
-	"golang.org/x/exp/maps"
-	"golang.org/x/exp/slices"
 )
 
 const (
@@ -90,6 +88,7 @@ var supportedHeaders = []string{
 	"X-Minio-Replication-Server-Side-Encryption-Iv",
 	"X-Minio-Replication-Encrypted-Multipart",
 	"X-Minio-Replication-Actual-Object-Size",
+	ReplicationSsecChecksumHeader,
 	// Add more supported headers here.
 }
 
@@ -110,6 +109,7 @@ var replicationToInternalHeaders = map[string]string{
 	"X-Minio-Replication-Server-Side-Encryption-Iv":             "X-Minio-Internal-Server-Side-Encryption-Iv",
 	"X-Minio-Replication-Encrypted-Multipart":                   "X-Minio-Internal-Encrypted-Multipart",
 	"X-Minio-Replication-Actual-Object-Size":                    "X-Minio-Internal-Actual-Object-Size",
+	ReplicationSsecChecksumHeader:                               ReplicationSsecChecksumHeader,
 	// Add more supported headers here.
 }
 
@@ -206,8 +206,8 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
 	for _, supportedHeader := range supportedHeaders {
 		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
 		if ok {
-			if slices.Contains(maps.Keys(replicationToInternalHeaders), supportedHeader) {
-				m[replicationToInternalHeaders[supportedHeader]] = strings.Join(value, ",")
+			if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
+				m[v] = strings.Join(value, ",")
 			} else {
 				m[supportedHeader] = strings.Join(value, ",")
 			}

cmd/http-tracer.go

@@ -142,6 +142,7 @@ func httpTracerMiddleware(h http.Handler) http.Handler {
 			Time:      reqStartTime,
 			Duration:  reqEndTime.Sub(respRecorder.StartTime),
 			Path:      reqPath,
+			Bytes:     int64(inputBytes + respRecorder.Size()),
 			HTTP: &madmin.TraceHTTPStats{
 				ReqInfo: madmin.TraceRequestInfo{
 					Time:     reqStartTime,

cmd/iam-etcd-store.go

@@ -236,9 +236,8 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 				// for the expiring credentials.
 				deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
 				deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
-				return nil
 			}
-			return err
+			return nil
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}

cmd/iam-object-store.go

@@ -25,7 +25,6 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 	"unicode/utf8"
 
@@ -34,6 +33,7 @@ import (
 	"github.com/minio/minio/internal/config"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
 	"github.com/puzpuzpuz/xsync/v3"
 )
 
@@ -44,8 +44,6 @@ type IAMObjectStore struct {
 
 	*iamCache
 
-	cachedIAMListing atomic.Value
-
 	usersSysType UsersSysType
 
 	objAPI ObjectLayer
@@ -256,9 +254,8 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 				// for the expiring credentials.
 				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
 				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
-				return nil
 			}
-			return err
+			return nil
 
 		}
 		u.Credentials.Claims = jwtClaims.Map()
@@ -421,8 +418,8 @@ func splitPath(s string, lastIndex bool) (string, string) {
 	return s[:i+1], s[i+1:]
 }
 
-func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
-	res := make(map[string][]string)
+func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (res map[string][]string, err error) {
+	res = make(map[string][]string)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) {
@@ -438,62 +435,10 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
 
 		res[listKey] = append(res[listKey], trimmedItem)
 	}
-	// Store the listing for later re-use.
-	iamOS.cachedIAMListing.Store(res)
+
 	return res, nil
 }
 
-// PurgeExpiredSTS - purge expired STS credentials from object store.
-func (iamOS *IAMObjectStore) PurgeExpiredSTS(ctx context.Context) error {
-	if iamOS.objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	bootstrapTraceMsg("purging expired STS credentials")
-
-	iamListing, ok := iamOS.cachedIAMListing.Load().(map[string][]string)
-	if !ok {
-		// There has been no store yet. This should never happen!
-		iamLogIf(GlobalContext, errors.New("WARNING: no cached IAM listing found"))
-		return nil
-	}
-
-	// Scan STS users on disk and purge expired ones. We do not need to hold a
-	// lock with store.lock() here.
-	stsAccountsFromStore := map[string]UserIdentity{}
-	stsAccPoliciesFromStore := xsync.NewMapOf[string, MappedPolicy]()
-	for _, item := range iamListing[stsListKey] {
-		userName := path.Dir(item)
-		// loadUser() will delete expired user during the load.
-		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
-		if err != nil && !errors.Is(err, errNoSuchUser) {
-			iamLogIf(GlobalContext,
-				fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item))
-		}
-
-	}
-	// Loading the STS policy mappings from disk ensures that stale entries
-	// (removed during loadUser() in the loop above) are removed from memory.
-	for _, item := range iamListing[policyDBSTSUsersListKey] {
-		stsName := strings.TrimSuffix(item, ".json")
-		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
-		if err != nil && !errors.Is(err, errNoSuchPolicy) {
-			iamLogIf(GlobalContext,
-				fmt.Errorf("unable to load policies during STS purge: %w (%s)", err, item))
-		}
-
-	}
-
-	// Store the newly populated map in the iam cache. This takes care of
-	// removing stale entries from the existing map.
-	iamOS.Lock()
-	defer iamOS.Unlock()
-	iamOS.iamCache.iamSTSAccountsMap = stsAccountsFromStore
-	iamOS.iamCache.iamSTSPolicyMap = stsAccPoliciesFromStore
-
-	return nil
-}
-
 // Assumes cache is locked by caller.
 func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
 	if iamOS.objAPI == nil {
@@ -594,6 +539,45 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	}
 
 	cache.buildUserGroupMemberships()
+
+	purgeStart := time.Now()
+
+	// Purge expired STS credentials.
+
+	// Scan STS users on disk and purge expired ones.
+	stsAccountsFromStore := map[string]UserIdentity{}
+	stsAccPoliciesFromStore := xsync.NewMapOf[string, MappedPolicy]()
+	for _, item := range listedConfigItems[stsListKey] {
+		userName := path.Dir(item)
+		// loadUser() will delete expired user during the load.
+		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
+		if err != nil && !errors.Is(err, errNoSuchUser) {
+			return fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item)
+		}
+
+	}
+	// Loading the STS policy mappings from disk ensures that stale entries
+	// (removed during loadUser() in the loop above) are removed from memory.
+	for _, item := range listedConfigItems[policyDBSTSUsersListKey] {
+		stsName := strings.TrimSuffix(item, ".json")
+		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
+		if err != nil && !errors.Is(err, errNoSuchPolicy) {
+			return fmt.Errorf("unable to load policies during STS purge: %w (%s)", err, item)
+		}
+
+	}
+
+	took := time.Since(purgeStart).Seconds()
+	if took > maxDurationSecondsForLog {
+		// Log if we took a lot of time to load.
+		logger.Info("IAM expired STS purge took %.2fs", took)
+	}
+
+	// Store the newly populated map in the iam cache. This takes care of
+	// removing stale entries from the existing map.
+	cache.iamSTSAccountsMap = stsAccountsFromStore
+	cache.iamSTSPolicyMap = stsAccPoliciesFromStore
+
 	return nil
 }
 

cmd/iam-store.go

@@ -33,8 +33,10 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
+	"github.com/minio/pkg/v3/console"
 	"github.com/minio/pkg/v3/policy"
 	"github.com/puzpuzpuz/xsync/v3"
+	"golang.org/x/sync/singleflight"
 )
 
 const (
@@ -532,16 +534,6 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
 	}
 }
 
-// PurgeExpiredSTS - purges expired STS credentials.
-func (store *IAMStoreSys) PurgeExpiredSTS(ctx context.Context) error {
-	iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore)
-	if !ok {
-		// No purging is done for non-object storage.
-		return nil
-	}
-	return iamOS.PurgeExpiredSTS(ctx)
-}
-
 // LoadIAMCache reads all IAM items and populates a new iamCache object and
 // replaces the in-memory cache object.
 func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) error {
@@ -643,6 +635,8 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
 // layer.
 type IAMStoreSys struct {
 	IAMStorageAPI
+
+	group *singleflight.Group
 }
 
 // HasWatcher - returns if the storage system has a watcher.
@@ -1810,6 +1804,7 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 		err = nil
 	}
 	delete(cache.iamUsersMap, accessKey)
+	store.group.Forget(accessKey)
 
 	cache.updatedAt = time.Now()
 
@@ -1885,6 +1880,7 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 			err := store.deleteUserIdentity(ctx, user, userType)
 			iamLogIf(GlobalContext, err)
 			delete(cache.iamUsersMap, user)
+			store.group.Forget(user)
 
 			deleted = true
 		}
@@ -2371,7 +2367,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 			return updatedAt, err
 		}
 
-		if len(policyBuf) > 2048 {
+		if len(policyBuf) > maxSVCSessionPolicySize {
 			return updatedAt, errSessionPolicyTooLarge
 		}
 
@@ -2572,69 +2568,88 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 }
 
 // LoadUser - attempts to load user info from storage and updates cache.
-func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
-	cache := store.lock()
-	defer store.unlock()
+func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error {
+	// We use singleflight to de-duplicate requests when server
+	// is coming up and loading accessKey and its associated assets
+	val, err, shared := store.group.Do(accessKey, func() (val interface{}, err error) {
+		cache := store.lock()
+		defer func() {
+			cache.updatedAt = time.Now()
+			store.unlock()
+		}()
 
-	cache.updatedAt = time.Now()
+		_, found := cache.iamUsersMap[accessKey]
 
-	_, found := cache.iamUsersMap[accessKey]
-
-	// Check for regular user access key
-	if !found {
-		store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)
-		if _, found = cache.iamUsersMap[accessKey]; found {
-			// load mapped policies
-			store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, cache.iamUserPolicyMap, 3)
-		}
-	}
-
-	// Check for service account
-	if !found {
-		store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
-		var svc UserIdentity
-		svc, found = cache.iamUsersMap[accessKey]
-		if found {
-			// Load parent user and mapped policies.
-			if store.getUsersSysType() == MinIOUsersSysType {
-				store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
-				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap, 3)
-			} else {
-				// In case of LDAP the parent user's policy mapping needs to be
-				// loaded into sts map
-				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
+		// Check for regular user access key
+		if !found {
+			err = store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)
+			if _, found = cache.iamUsersMap[accessKey]; found {
+				// load mapped policies
+				err = store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, cache.iamUserPolicyMap, 3)
 			}
 		}
-	}
 
-	// Check for STS account
-	stsAccountFound := false
-	var stsUserCred UserIdentity
-	if !found {
-		store.loadUser(ctx, accessKey, stsUser, cache.iamSTSAccountsMap)
-		if stsUserCred, found = cache.iamSTSAccountsMap[accessKey]; found {
-			// Load mapped policy
-			store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
-			stsAccountFound = true
-		}
-	}
-
-	// Load any associated policy definitions
-	if !stsAccountFound {
-		pols, _ := cache.iamUserPolicyMap.Load(accessKey)
-		for _, policy := range pols.toSlice() {
-			if _, found = cache.iamPolicyDocsMap[policy]; !found {
-				store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
+		// Check for service account
+		if !found {
+			err = store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
+			var svc UserIdentity
+			svc, found = cache.iamUsersMap[accessKey]
+			if found {
+				// Load parent user and mapped policies.
+				if store.getUsersSysType() == MinIOUsersSysType {
+					err = store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
+					if err == nil {
+						err = store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap, 3)
+					}
+				} else {
+					// In case of LDAP the parent user's policy mapping needs to be
+					// loaded into sts map
+					err = store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
+				}
 			}
 		}
-	} else {
-		pols, _ := cache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
-		for _, policy := range pols.toSlice() {
-			if _, found = cache.iamPolicyDocsMap[policy]; !found {
-				store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
+
+		// Check for STS account
+		stsAccountFound := false
+		var stsUserCred UserIdentity
+		if !found {
+			err = store.loadUser(ctx, accessKey, stsUser, cache.iamSTSAccountsMap)
+			if stsUserCred, found = cache.iamSTSAccountsMap[accessKey]; found {
+				// Load mapped policy
+				err = store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
+				stsAccountFound = true
 			}
 		}
+
+		// Load any associated policy definitions
+		if !stsAccountFound {
+			pols, _ := cache.iamUserPolicyMap.Load(accessKey)
+			for _, policy := range pols.toSlice() {
+				if _, found = cache.iamPolicyDocsMap[policy]; !found {
+					err = store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
+				}
+			}
+		} else {
+			pols, _ := cache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
+			for _, policy := range pols.toSlice() {
+				if _, found = cache.iamPolicyDocsMap[policy]; !found {
+					err = store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
+				}
+			}
+		}
+
+		return "done", err
+	})
+
+	if serverDebugLog {
+		console.Debugln("loadUser: loading shared", val, err, shared)
 	}
+
+	if IsErr(err, errNoSuchUser, errNoSuchPolicy, errNoSuchGroup) {
+		return nil
+	}
+
+	return err
 }
 
 func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {

cmd/iam.go

@@ -52,6 +52,7 @@ import (
 	"github.com/minio/pkg/v3/ldap"
 	"github.com/minio/pkg/v3/policy"
 	etcd "go.etcd.io/etcd/client/v3"
+	"golang.org/x/sync/singleflight"
 )
 
 // UsersSysType - defines the type of users and groups system that is
@@ -78,6 +79,10 @@ const (
 	inheritedPolicyType = "inherited-policy"
 )
 
+const (
+	maxSVCSessionPolicySize = 4096
+)
+
 // IAMSys - config system.
 type IAMSys struct {
 	// Need to keep them here to keep alignment - ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
@@ -173,9 +178,9 @@ func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
 	}
 
 	if etcdClient == nil {
-		sys.store = &IAMStoreSys{newIAMObjectStore(objAPI, sys.usersSysType)}
+		sys.store = &IAMStoreSys{newIAMObjectStore(objAPI, sys.usersSysType), &singleflight.Group{}}
 	} else {
-		sys.store = &IAMStoreSys{newIAMEtcdStore(etcdClient, sys.usersSysType)}
+		sys.store = &IAMStoreSys{newIAMEtcdStore(etcdClient, sys.usersSysType), &singleflight.Group{}}
 	}
 }
 
@@ -348,6 +353,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	bootstrapTraceMsg("finishing IAM loading")
 }
 
+const maxDurationSecondsForLog = 5
+
 func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Duration) {
 	// Watch for IAM config changes for iamStorageWatcher.
 	watcher, isWatcher := sys.store.IAMStorageAPI.(iamStorageWatcher)
@@ -380,7 +387,6 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 		return baseInterval/2 + randAmt
 	}
 
-	var maxDurationSecondsForLog float64 = 5
 	timer := time.NewTimer(waitInterval())
 	defer timer.Stop()
 
@@ -399,18 +405,6 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 				}
 			}
 
-			// Purge expired STS credentials.
-			purgeStart := time.Now()
-			if err := sys.store.PurgeExpiredSTS(ctx); err != nil {
-				iamLogIf(ctx, fmt.Errorf("Failure in periodic STS purge for IAM (took %.2fs): %v", time.Since(purgeStart).Seconds(), err))
-			} else {
-				took := time.Since(purgeStart).Seconds()
-				if took > maxDurationSecondsForLog {
-					// Log if we took a lot of time to load.
-					logger.Info("IAM expired STS purge took %.2fs", took)
-				}
-			}
-
 			// The following actions are performed about once in 4 times that
 			// IAM is refreshed:
 			if r.Intn(4) == 0 {
@@ -977,7 +971,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		if err != nil {
 			return auth.Credentials{}, time.Time{}, err
 		}
-		if len(policyBuf) > 2048 {
+		if len(policyBuf) > maxSVCSessionPolicySize {
 			return auth.Credentials{}, time.Time{}, errSessionPolicyTooLarge
 		}
 	}
@@ -1574,31 +1568,16 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 
 func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet {
 	entityKeysInStorage := set.NewStringSet()
-	if iamOS, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
-		// Load existing mapping keys from the cached listing for
-		// `IAMObjectStore`.
-		iamFilesListing := iamOS.cachedIAMListing.Load().(map[string][]string)
-		listKey := policyDBSTSUsersListKey
-		if isGroup {
-			listKey = policyDBGroupsListKey
-		}
-		for _, item := range iamFilesListing[listKey] {
-			stsUserName := strings.TrimSuffix(item, ".json")
-			entityKeysInStorage.Add(stsUserName)
-		}
-	} else {
-		// For non-iam object store, we copy the mapping keys from the cache.
-		cache := sys.store.rlock()
-		defer sys.store.runlock()
-		cachedPolicyMap := cache.iamSTSPolicyMap
-		if isGroup {
-			cachedPolicyMap = cache.iamGroupPolicyMap
-		}
-		cachedPolicyMap.Range(func(k string, v MappedPolicy) bool {
-			entityKeysInStorage.Add(k)
-			return true
-		})
+	cache := sys.store.rlock()
+	defer sys.store.runlock()
+	cachedPolicyMap := cache.iamSTSPolicyMap
+	if isGroup {
+		cachedPolicyMap = cache.iamGroupPolicyMap
 	}
+	cachedPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		entityKeysInStorage.Add(k)
+		return true
+	})
 
 	return entityKeysInStorage
 }
@@ -1724,31 +1703,46 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 	return nil
 }
 
-// GetUser - get user credentials
-func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentity, ok bool) {
+// CheckKey validates the incoming accessKey
+func (sys *IAMSys) CheckKey(ctx context.Context, accessKey string) (u UserIdentity, ok bool, err error) {
 	if !sys.Initialized() {
-		return u, false
+		return u, false, nil
 	}
 
 	if accessKey == globalActiveCred.AccessKey {
-		return newUserIdentity(globalActiveCred), true
+		return newUserIdentity(globalActiveCred), true, nil
 	}
 
 	loadUserCalled := false
 	select {
 	case <-sys.configLoaded:
 	default:
-		sys.store.LoadUser(ctx, accessKey)
+		err = sys.store.LoadUser(ctx, accessKey)
 		loadUserCalled = true
 	}
 
 	u, ok = sys.store.GetUser(accessKey)
 	if !ok && !loadUserCalled {
-		sys.store.LoadUser(ctx, accessKey)
+		err = sys.store.LoadUser(ctx, accessKey)
+		loadUserCalled = true
+
 		u, ok = sys.store.GetUser(accessKey)
 	}
 
-	return u, ok && u.Credentials.IsValid()
+	if !ok && loadUserCalled && err != nil {
+		iamLogOnceIf(ctx, err, accessKey)
+
+		// return 503 to application
+		return u, false, errIAMNotInitialized
+	}
+
+	return u, ok && u.Credentials.IsValid(), nil
+}
+
+// GetUser - get user credentials
+func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentity, ok bool) {
+	u, ok, _ = sys.CheckKey(ctx, accessKey)
+	return u, ok
 }
 
 // Notify all other MinIO peers to load group.
@@ -1982,20 +1976,22 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		}
 		isGroup = false
 	} else {
-		if isAttach {
-			var underBaseDN bool
-			if dnResult, underBaseDN, err = sys.LDAPConfig.GetValidatedGroupDN(nil, r.Group); err != nil {
-				iamLogIf(ctx, err)
-				return
-			} else if dnResult == nil || !underBaseDN {
+		var underBaseDN bool
+		if dnResult, underBaseDN, err = sys.LDAPConfig.GetValidatedGroupDN(nil, r.Group); err != nil {
+			iamLogIf(ctx, err)
+			return
+		}
+		if dnResult == nil || !underBaseDN {
+			if !isAttach {
+				dn = r.Group
+			} else {
 				err = errNoSuchGroup
 				return
 			}
+		} else {
 			// We use the group DN returned by the LDAP server (this may not
 			// equal the input group name, but we assume it is canonical).
 			dn = dnResult.NormDN
-		} else {
-			dn = r.Group
 		}
 		isGroup = true
 	}

cmd/logging.go

@@ -8,6 +8,10 @@ import (
 	"github.com/minio/minio/internal/logger"
 )
 
+func proxyLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "proxy", err, errKind...)
+}
+
 func replLogIf(ctx context.Context, err error, errKind ...interface{}) {
 	logger.LogIf(ctx, "replication", err, errKind...)
 }
@@ -16,6 +20,10 @@ func replLogOnceIf(ctx context.Context, err error, id string, errKind ...interfa
 	logger.LogOnceIf(ctx, "replication", err, id, errKind...)
 }
 
+func iamLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "iam", err, id, errKind...)
+}
+
 func iamLogIf(ctx context.Context, err error, errKind ...interface{}) {
 	if !errors.Is(err, grid.ErrDisconnected) {
 		logger.LogIf(ctx, "iam", err, errKind...)

cmd/metacache-entries.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"os"
 	"path"
 	"sort"
@@ -301,7 +300,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro
 		}, nil
 	}
 	// Too small gains to reuse cache here.
-	return getFileInfoVersions(e.metadata, bucket, e.name, false)
+	return getFileInfoVersions(e.metadata, bucket, e.name, false, true)
 }
 
 // metaCacheEntries is a slice of metacache entries.
@@ -385,9 +384,6 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 		// shallow decode.
 		xl, err := entry.xlmeta()
 		if err != nil {
-			if !errors.Is(err, errFileNotFound) {
-				internalLogIf(GlobalContext, err)
-			}
 			continue
 		}
 		objsValid++
@@ -752,7 +748,9 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
 				if !dirMatches {
 					// We have an object `name` or 'name/' and a directory `name/`.
 					if other.isDir() {
-						console.Debugln("mergeEntryChannels: discarding directory", other.name, "for object", best.name)
+						if serverDebugLog {
+							console.Debugln("mergeEntryChannels: discarding directory", other.name, "for object", best.name)
+						}
 						// Discard the directory.
 						if err := selectFrom(otherIdx); err != nil {
 							return err
@@ -760,7 +758,9 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
 						continue
 					}
 					// Replace directory with object.
-					console.Debugln("mergeEntryChannels: discarding directory", best.name, "for object", other.name)
+					if serverDebugLog {
+						console.Debugln("mergeEntryChannels: discarding directory", best.name, "for object", other.name)
+					}
 					toMerge = toMerge[:0]
 					best = other
 					bestIdx = otherIdx

cmd/metacache-set.go

@@ -391,7 +391,7 @@ func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSor
 			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() {
 				return true
 			}
-			if entry.isAllFreeVersions() {
+			if !o.InclDeleted && entry.isAllFreeVersions() {
 				return true
 			}
 			entries.o = append(entries.o, entry)

cmd/metacache-walk.go

@@ -405,7 +405,7 @@ func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions,
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.WalkDir(ctx, opts, wr)
 }

cmd/metrics-realtime.go

@@ -132,6 +132,15 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 			m.Aggregated.CPU.LoadStat = loadStat
 		}
 	}
+	if types.Contains(madmin.MetricsRPC) {
+		gr := globalGrid.Load()
+		if gr == nil {
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: Grid not initialized", byHostName))
+		} else {
+			stats := gr.ConnStats()
+			m.Aggregated.RPC = &stats
+		}
+	}
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.

cmd/metrics-v2.go

@@ -273,13 +273,10 @@ const (
 	vmemory          = "virtual_memory_bytes"
 	cpu              = "cpu_total_seconds"
 
-	expiryPendingTasks           MetricName = "expiry_pending_tasks"
 	expiryMissedTasks            MetricName = "expiry_missed_tasks"
 	expiryMissedFreeVersions     MetricName = "expiry_missed_freeversions"
 	expiryMissedTierJournalTasks MetricName = "expiry_missed_tierjournal_tasks"
 	expiryNumWorkers             MetricName = "expiry_num_workers"
-	transitionPendingTasks       MetricName = "transition_pending_tasks"
-	transitionActiveTasks        MetricName = "transition_active_tasks"
 	transitionMissedTasks        MetricName = "transition_missed_immediate_tasks"
 
 	transitionedBytes    MetricName = "transitioned_bytes"
@@ -820,7 +817,7 @@ func getClusterObjectVersionsMD() MetricDescription {
 
 func getClusterRepLinkLatencyCurrMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      currLinkLatency,
 		Help:      "Replication current link latency in milliseconds",
@@ -830,7 +827,7 @@ func getClusterRepLinkLatencyCurrMD() MetricDescription {
 
 func getClusterRepLinkOnlineMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      linkOnline,
 		Help:      "Reports whether replication link is online (1) or offline(0)",
@@ -840,7 +837,7 @@ func getClusterRepLinkOnlineMD() MetricDescription {
 
 func getClusterRepLinkCurrOfflineDurationMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      linkOfflineDuration,
 		Help:      "Duration of replication link being offline in seconds since last offline event",
@@ -850,7 +847,7 @@ func getClusterRepLinkCurrOfflineDurationMD() MetricDescription {
 
 func getClusterRepLinkTotalOfflineDurationMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      linkDowntimeTotalDuration,
 		Help:      "Total downtime of replication link in seconds since server uptime",
@@ -978,7 +975,7 @@ func getRepReceivedOperationsMD(namespace MetricNamespace) MetricDescription {
 
 func getClusterReplMRFFailedOperationsMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      recentBacklogCount,
 		Help:      "Total number of objects seen in replication backlog in the last 5 minutes",
@@ -998,7 +995,7 @@ func getClusterRepCredentialErrorsMD(namespace MetricNamespace) MetricDescriptio
 
 func getClusterReplCurrQueuedOperationsMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      currInQueueCount,
 		Help:      "Total number of objects queued for replication in the last full minute",
@@ -1008,7 +1005,7 @@ func getClusterReplCurrQueuedOperationsMD() MetricDescription {
 
 func getClusterReplCurrQueuedBytesMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      currInQueueBytes,
 		Help:      "Total number of bytes queued for replication in the last full minute",
@@ -1018,7 +1015,7 @@ func getClusterReplCurrQueuedBytesMD() MetricDescription {
 
 func getClusterReplActiveWorkersCountMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      currActiveWorkers,
 		Help:      "Total number of active replication workers",
@@ -1028,7 +1025,7 @@ func getClusterReplActiveWorkersCountMD() MetricDescription {
 
 func getClusterReplAvgActiveWorkersCountMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      avgActiveWorkers,
 		Help:      "Average number of active replication workers",
@@ -1038,7 +1035,7 @@ func getClusterReplAvgActiveWorkersCountMD() MetricDescription {
 
 func getClusterReplMaxActiveWorkersCountMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      maxActiveWorkers,
 		Help:      "Maximum number of active replication workers seen since server uptime",
@@ -1048,7 +1045,7 @@ func getClusterReplMaxActiveWorkersCountMD() MetricDescription {
 
 func getClusterReplCurrentTransferRateMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      currTransferRate,
 		Help:      "Current replication transfer rate in bytes/sec",
@@ -1058,7 +1055,7 @@ func getClusterReplCurrentTransferRateMD() MetricDescription {
 
 func getClusterRepLinkLatencyMaxMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      maxLinkLatency,
 		Help:      "Maximum replication link latency in milliseconds seen since server uptime",
@@ -1068,7 +1065,7 @@ func getClusterRepLinkLatencyMaxMD() MetricDescription {
 
 func getClusterRepLinkLatencyAvgMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      avgLinkLatency,
 		Help:      "Average replication link latency in milliseconds",
@@ -1078,7 +1075,7 @@ func getClusterRepLinkLatencyAvgMD() MetricDescription {
 
 func getClusterReplAvgQueuedOperationsMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      avgInQueueCount,
 		Help:      "Average number of objects queued for replication since server uptime",
@@ -1088,7 +1085,7 @@ func getClusterReplAvgQueuedOperationsMD() MetricDescription {
 
 func getClusterReplAvgQueuedBytesMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      avgInQueueBytes,
 		Help:      "Average number of bytes queued for replication since server uptime",
@@ -1098,7 +1095,7 @@ func getClusterReplAvgQueuedBytesMD() MetricDescription {
 
 func getClusterReplMaxQueuedOperationsMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      maxInQueueCount,
 		Help:      "Maximum number of objects queued for replication since server uptime",
@@ -1108,7 +1105,7 @@ func getClusterReplMaxQueuedOperationsMD() MetricDescription {
 
 func getClusterReplMaxQueuedBytesMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      maxInQueueBytes,
 		Help:      "Maximum number of bytes queued for replication since server uptime",
@@ -1118,7 +1115,7 @@ func getClusterReplMaxQueuedBytesMD() MetricDescription {
 
 func getClusterReplAvgTransferRateMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      avgTransferRate,
 		Help:      "Average replication transfer rate in bytes/sec",
@@ -1128,7 +1125,7 @@ func getClusterReplAvgTransferRateMD() MetricDescription {
 
 func getClusterReplMaxTransferRateMD() MetricDescription {
 	return MetricDescription{
-		Namespace: clusterMetricNamespace,
+		Namespace: nodeMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      maxTransferRate,
 		Help:      "Maximum replication transfer rate in bytes/sec seen since server uptime",

cmd/metrics-v3-ilm.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2024 MinIO, Inc.
+//
+// # This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+)
+
+const (
+	expiryPendingTasks             = "expiry_pending_tasks"
+	transitionActiveTasks          = "transition_active_tasks"
+	transitionPendingTasks         = "transition_pending_tasks"
+	transitionMissedImmediateTasks = "transition_missed_immediate_tasks"
+	versionsScanned                = "versions_scanned"
+)
+
+var (
+	ilmExpiryPendingTasksMD             = NewGaugeMD(expiryPendingTasks, "Number of pending ILM expiry tasks in the queue")
+	ilmTransitionActiveTasksMD          = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks")
+	ilmTransitionPendingTasksMD         = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue")
+	ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks")
+	ilmVersionsScannedMD                = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start")
+)
+
+// loadILMMetrics - `MetricsLoaderFn` for ILM metrics.
+func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
+	if globalExpiryState != nil {
+		m.Set(expiryPendingTasks, float64(globalExpiryState.PendingTasks()))
+	}
+	if globalTransitionState != nil {
+		m.Set(transitionActiveTasks, float64(globalTransitionState.ActiveTasks()))
+		m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks()))
+		m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks()))
+	}
+	m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM)))
+
+	return nil
+}

cmd/metrics-v3.go

@@ -54,6 +54,7 @@ const (
 	clusterIAMCollectorPath          collectorPath = "/cluster/iam"
 	clusterConfigCollectorPath       collectorPath = "/cluster/config"
 
+	ilmCollectorPath           collectorPath = "/ilm"
 	auditCollectorPath         collectorPath = "/audit"
 	loggerWebhookCollectorPath collectorPath = "/logger/webhook"
 	replicationCollectorPath   collectorPath = "/replication"
@@ -382,6 +383,17 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadAuditMetrics,
 	)
 
+	ilmMG := NewMetricsGroup(ilmCollectorPath,
+		[]MetricDescriptor{
+			ilmExpiryPendingTasksMD,
+			ilmTransitionActiveTasksMD,
+			ilmTransitionPendingTasksMD,
+			ilmTransitionMissedImmediateTasksMD,
+			ilmVersionsScannedMD,
+		},
+		loadILMMetrics,
+	)
+
 	allMetricGroups := []*MetricsGroup{
 		apiRequestsMG,
 		bucketAPIMG,
@@ -402,6 +414,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		clusterReplicationMG,
 		clusterConfigMG,
 
+		ilmMG,
 		scannerMG,
 		auditMG,
 		loggerWebhookMG,

cmd/naughty-disk_test.go

@@ -215,9 +215,9 @@ func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVol
 	return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
 }
 
-func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
+func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
 	if err := d.calcError(); err != nil {
-		return err
+		return nil, err
 	}
 	return d.disk.CheckParts(ctx, volume, path, fi)
 }
@@ -289,9 +289,9 @@ func (d *naughtyDisk) ReadXL(ctx context.Context, volume string, path string, re
 	return d.disk.ReadXL(ctx, volume, path, readData)
 }
 
-func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
+func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
 	if err := d.calcError(); err != nil {
-		return err
+		return nil, err
 	}
 	return d.disk.VerifyFile(ctx, volume, path, fi)
 }

cmd/object-api-datatypes.go

@@ -233,7 +233,7 @@ func (o ObjectInfo) ExpiresStr() string {
 
 // ArchiveInfo returns any saved zip archive meta information.
 // It will be decrypted if needed.
-func (o *ObjectInfo) ArchiveInfo() []byte {
+func (o *ObjectInfo) ArchiveInfo(h http.Header) []byte {
 	if len(o.UserDefined) == 0 {
 		return nil
 	}
@@ -243,7 +243,7 @@ func (o *ObjectInfo) ArchiveInfo() []byte {
 	}
 	data := []byte(z)
 	if v, ok := o.UserDefined[archiveTypeMetadataKey]; ok && v == archiveTypeEnc {
-		decrypted, err := o.metadataDecrypter()(archiveTypeEnc, data)
+		decrypted, err := o.metadataDecrypter(h)(archiveTypeEnc, data)
 		if err != nil {
 			encLogIf(GlobalContext, err)
 			return nil
@@ -326,6 +326,7 @@ func (ri ReplicateObjectInfo) ToObjectInfo() ObjectInfo {
 		VersionPurgeStatusInternal: ri.VersionPurgeStatusInternal,
 		DeleteMarker:               true,
 		UserDefined:                map[string]string{},
+		Checksum:                   ri.Checksum,
 	}
 }
 
@@ -357,6 +358,7 @@ type ReplicateObjectInfo struct {
 	TargetStatuses       map[string]replication.StatusType
 	TargetPurgeStatuses  map[string]VersionPurgeStatusType
 	ReplicationTimestamp time.Time
+	Checksum             []byte
 }
 
 // MultipartInfo captures metadata information about the uploadId

cmd/object-api-datatypes_gen.go

@@ -1896,9 +1896,9 @@ func (z *PartInfo) Msgsize() (s int) {
 // MarshalMsg implements msgp.Marshaler
 func (z *ReplicateObjectInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 25
+	// map header, size 26
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x1a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -2012,6 +2012,9 @@ func (z *ReplicateObjectInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ReplicationTimestamp"
 	o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
 	o = msgp.AppendTime(o, z.ReplicationTimestamp)
+	// string "Checksum"
+	o = append(o, 0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d)
+	o = msgp.AppendBytes(o, z.Checksum)
 	return
 }
 
@@ -2231,6 +2234,12 @@ func (z *ReplicateObjectInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ReplicationTimestamp")
 				return
 			}
+		case "Checksum":
+			z.Checksum, bts, err = msgp.ReadBytesBytes(bts, z.Checksum)
+			if err != nil {
+				err = msgp.WrapError(err, "Checksum")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -2259,7 +2268,7 @@ func (z *ReplicateObjectInfo) Msgsize() (s int) {
 			s += msgp.StringPrefixSize + len(za0003) + za0004.Msgsize()
 		}
 	}
-	s += 21 + msgp.TimeSize
+	s += 21 + msgp.TimeSize + 9 + msgp.BytesPrefixSize + len(z.Checksum)
 	return
 }
 

cmd/object-api-options.go

@@ -37,6 +37,15 @@ func getDefaultOpts(header http.Header, copySource bool, metadata map[string]str
 	var sse encrypt.ServerSide
 
 	opts = ObjectOptions{UserDefined: metadata}
+	if v, ok := header[xhttp.MinIOSourceProxyRequest]; ok {
+		opts.ProxyHeaderSet = true
+		opts.ProxyRequest = strings.Join(v, "") == "true"
+	}
+	if _, ok := header[xhttp.MinIOSourceReplicationRequest]; ok {
+		opts.ReplicationRequest = true
+	}
+	opts.Speedtest = header.Get(globalObjectPerfUserMetadata) != ""
+
 	if copySource {
 		if crypto.SSECopy.IsRequested(header) {
 			clientKey, err = crypto.SSECopy.ParseHTTP(header)
@@ -66,14 +75,7 @@ func getDefaultOpts(header http.Header, copySource bool, metadata map[string]str
 	if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
 		opts.ServerSideEncryption = encrypt.NewSSE()
 	}
-	if v, ok := header[xhttp.MinIOSourceProxyRequest]; ok {
-		opts.ProxyHeaderSet = true
-		opts.ProxyRequest = strings.Join(v, "") == "true"
-	}
-	if _, ok := header[xhttp.MinIOSourceReplicationRequest]; ok {
-		opts.ReplicationRequest = true
-	}
-	opts.Speedtest = header.Get(globalObjectPerfUserMetadata) != ""
+
 	return
 }
 
@@ -494,5 +496,8 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
 	if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok {
 		opts.ReplicationRequest = true
 	}
+	if r.Header.Get(ReplicationSsecChecksumHeader) != "" {
+		opts.UserDefined[ReplicationSsecChecksumHeader] = r.Header.Get(ReplicationSsecChecksumHeader)
+	}
 	return opts, nil
 }

cmd/object-api-utils.go

@@ -80,6 +80,18 @@ const (
 	compMinIndexSize = 8 << 20
 )
 
+// getkeyeparator - returns the separator to be used for
+// persisting on drive.
+//
+// - ":" is used on non-windows platforms
+// - "_" is used on windows platforms
+func getKeySeparator() string {
+	if runtime.GOOS == globalWindowsOSName {
+		return "_"
+	}
+	return ":"
+}
+
 // isMinioBucket returns true if given bucket is a MinIO internal
 // bucket and false otherwise.
 func isMinioMetaBucketName(bucket string) bool {
@@ -755,7 +767,7 @@ type ObjReaderFn func(inputReader io.Reader, h http.Header, cleanupFns ...func()
 // are called on Close() in FIFO order as passed in ObjReadFn(). NOTE: It is
 // assumed that clean up functions do not panic (otherwise, they may
 // not all run!).
-func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
+func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, h http.Header) (
 	fn ObjReaderFn, off, length int64, err error,
 ) {
 	if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
@@ -810,7 +822,9 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 				return b, nil
 			}
 			if isEncrypted {
-				decrypt = oi.compressionIndexDecrypt
+				decrypt = func(b []byte) ([]byte, error) {
+					return oi.compressionIndexDecrypt(b, h)
+				}
 			}
 			// In case of range based queries on multiparts, the offset and length are reduced.
 			off, decOff, firstPart, decryptSkip, seqNum = getCompressedOffsets(oi, off, decrypt)
@@ -982,8 +996,8 @@ func compressionIndexEncrypter(key crypto.ObjectKey, input func() []byte) func()
 }
 
 // compressionIndexDecrypt reverses compressionIndexEncrypter.
-func (o *ObjectInfo) compressionIndexDecrypt(input []byte) ([]byte, error) {
-	return o.metadataDecrypter()("compression-index", input)
+func (o *ObjectInfo) compressionIndexDecrypt(input []byte, h http.Header) ([]byte, error) {
+	return o.metadataDecrypter(h)("compression-index", input)
 }
 
 // SealMD5CurrFn seals md5sum with object encryption key and returns sealed
@@ -1206,7 +1220,20 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 	}
 
 	if nDisks < len(di)/2 || nDisks <= 0 {
-		return false, fmt.Errorf("not enough online disks to calculate the available space, expected (%d)/(%d)", (len(di)/2)+1, nDisks)
+		var errs []error
+		for index, disk := range di {
+			switch {
+			case disk == nil:
+				errs = append(errs, fmt.Errorf("disk[%d]: offline", index))
+			case disk.Error != "":
+				errs = append(errs, fmt.Errorf("disk %s: %s", disk.Endpoint, disk.Error))
+			case disk.Total == 0:
+				errs = append(errs, fmt.Errorf("disk %s: total is zero", disk.Endpoint))
+			}
+		}
+		// Log disk errors.
+		peersLogIf(context.Background(), errors.Join(errs...))
+		return false, fmt.Errorf("not enough online disks to calculate the available space, need %d, found %d", (len(di)/2)+1, nDisks)
 	}
 
 	// Check we have enough on each disk, ignoring diskFillFraction.

cmd/object-handlers-common.go

@@ -335,7 +335,7 @@ func isETagEqual(left, right string) bool {
 // setPutObjHeaders sets all the necessary headers returned back
 // upon a success Put/Copy/CompleteMultipart/Delete requests
 // to activate delete only headers set delete as true
-func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
+func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool, h http.Header) {
 	// We must not use the http.Header().Set method here because some (broken)
 	// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
 	// Therefore, we have to set the ETag directly as map entry.
@@ -357,7 +357,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
 			lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 		}
 	}
-	hash.AddChecksumHeader(w, objInfo.decryptChecksums(0))
+	hash.AddChecksumHeader(w, objInfo.decryptChecksums(0, h))
 }
 
 func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete, lcEvent lifecycle.Event) {

cmd/object-handlers.go

@@ -384,6 +384,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	}
 
 	cachedResult := globalCacheConfig.Enabled() && opts.VersionID == ""
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		// No need to check cache for encrypted objects.
+		cachedResult = false
+	}
 
 	var update bool
 	if cachedResult {
@@ -606,12 +610,14 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
 			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 		}
+		// Never store encrypted objects in cache.
+		update = false
 		objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
 	}
 
 	if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
 		// AWS S3 silently drops checksums on range requests.
-		hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber))
+		hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber, r.Header))
 	}
 
 	var buf *bytebufferpool.ByteBuffer
@@ -758,7 +764,7 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
 	w.Header().Del(xhttp.ContentType)
 
 	if _, ok := opts.ObjectAttributes[xhttp.Checksum]; ok {
-		chkSums := objInfo.decryptChecksums(0)
+		chkSums := objInfo.decryptChecksums(0, r.Header)
 		// AWS does not appear to append part number on this API call.
 		switch {
 		case chkSums["CRC32"] != "":
@@ -789,7 +795,7 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
 		OA.StorageClass = filterStorageClass(ctx, objInfo.StorageClass)
 	}
 
-	objInfo.decryptPartsChecksums()
+	objInfo.decryptPartsChecksums(r.Header)
 
 	if _, ok := opts.ObjectAttributes[xhttp.ObjectParts]; ok {
 		OA.ObjectParts = new(objectAttributesParts)
@@ -949,8 +955,10 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 	}
 
 	cachedResult := globalCacheConfig.Enabled() && opts.VersionID == ""
-
-	var update bool
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		// No need to check cache for encrypted objects.
+		cachedResult = false
+	}
 	if cachedResult {
 		rc := &cache.CondCheck{}
 		h := r.Header.Clone()
@@ -959,7 +967,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		}
 		rc.Init(bucket, object, h)
 
-		ci, err := globalCacheConfig.Get(rc)
+		ci, _ := globalCacheConfig.Get(rc)
 		if ci != nil {
 			tgs, ok := ci.Metadata[xhttp.AmzObjectTagging]
 			if ok {
@@ -1018,9 +1026,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 				return
 			}
 		}
-		if errors.Is(err, cache.ErrKeyMissing) {
-			update = true
-		}
 	}
 
 	opts.FastGetObjInfo = true
@@ -1124,24 +1129,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		return
 	}
 
-	if update {
-		asize, err := objInfo.GetActualSize()
-		if err != nil {
-			asize = objInfo.Size
-		}
-
-		defer globalCacheConfig.Set(&cache.ObjectInfo{
-			Key:          objInfo.Name,
-			Bucket:       objInfo.Bucket,
-			ETag:         objInfo.ETag,
-			ModTime:      objInfo.ModTime,
-			Expires:      objInfo.ExpiresStr(),
-			CacheControl: objInfo.CacheControl,
-			Size:         asize,
-			Metadata:     cleanReservedKeys(objInfo.UserDefined),
-		})
-	}
-
 	// Validate pre-conditions if any.
 	if checkPreconditions(ctx, w, r, objInfo, opts) {
 		return
@@ -1169,7 +1156,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 
 	if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
 		// AWS S3 silently drops checksums on range requests.
-		hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber))
+		hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber, r.Header))
 	}
 
 	// Set standard object headers.
@@ -1469,9 +1456,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// convert copy src encryption options for GET calls
 	getOpts := ObjectOptions{
-		VersionID:        srcOpts.VersionID,
-		Versioned:        srcOpts.Versioned,
-		VersionSuspended: srcOpts.VersionSuspended,
+		VersionID:          srcOpts.VersionID,
+		Versioned:          srcOpts.Versioned,
+		VersionSuspended:   srcOpts.VersionSuspended,
+		ReplicationRequest: r.Header.Get(xhttp.MinIOSourceReplicationRequest) == "true",
 	}
 	getSSE := encrypt.SSE(srcOpts.ServerSideEncryption)
 	if getSSE != srcOpts.ServerSideEncryption {
@@ -1567,7 +1555,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// Pass the decompressed stream to such calls.
 	isDstCompressed := isCompressible(r.Header, dstObject) &&
 		length > minCompressibleSize &&
-		!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) && !cpSrcDstSame && !objectEncryption
+		!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) && !objectEncryption
 	if isDstCompressed {
 		compressMetadata = make(map[string]string, 2)
 		// Preserving the compression metadata.
@@ -1603,7 +1591,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 	// Encryption parameters not present for this object.
-	if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) {
+	if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) && r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidSSECustomerAlgorithm), r.URL)
 		return
 	}
@@ -1929,7 +1917,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 	}
 
-	setPutObjHeaders(w, objInfo, false)
+	setPutObjHeaders(w, objInfo, false, r.Header)
 	// We must not use the http.Header().Set method here because some (broken)
 	// clients expect the x-amz-copy-source-version-id header key to be literally
 	// "x-amz-copy-source-version-id"- not in canonicalized form, preserve it.
@@ -2139,7 +2127,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	})
 
 	var buf *bytebufferpool.ByteBuffer
-	if globalCacheConfig.MatchesSize(size) {
+	// Disable cache for encrypted objects - headers are applied with sseConfig.Apply if auto encrypted.
+	if globalCacheConfig.MatchesSize(size) && !crypto.Requested(r.Header) {
 		buf = bytebufferpool.Get()
 		defer bytebufferpool.Put(buf)
 	}
@@ -2262,11 +2251,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			return
 		}
 
-		if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL)
-			return
-		}
-
 		reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -2351,7 +2335,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 	}
 
-	setPutObjHeaders(w, objInfo, false)
+	setPutObjHeaders(w, objInfo, false, r.Header)
 
 	defer func() {
 		var data []byte
@@ -2943,7 +2927,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 
 	defer globalCacheConfig.Delete(bucket, object)
 
-	setPutObjHeaders(w, objInfo, true)
+	setPutObjHeaders(w, objInfo, true, r.Header)
 	writeSuccessNoContent(w)
 
 	eventName := event.ObjectRemovedDelete

cmd/object-handlers_test.go

@@ -61,7 +61,7 @@ const (
 
 // Wrapper for calling HeadObject API handler tests for both Erasure multiple disks and FS single drive setup.
 func TestAPIHeadObjectHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testAPIHeadObjectHandler, []string{"HeadObject"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIHeadObjectHandler, endpoints: []string{"HeadObject"}})
 }
 
 func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -207,7 +207,7 @@ func TestAPIHeadObjectHandlerWithEncryption(t *testing.T) {
 	defer func() { globalPolicySys = nil }()
 
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIHeadObjectHandlerWithEncryption, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject", "HeadObject"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIHeadObjectHandlerWithEncryption, endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject", "HeadObject"}})
 }
 
 func testAPIHeadObjectHandlerWithEncryption(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -707,7 +707,13 @@ func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName str
 		{"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)},
 		{"enc-mp-3", []int64{10499807, 10499963, 7}, mapCopy(metaWithSSEC)},
 	}
-
+	// SSEC can't be used with compression
+	globalCompressConfigMu.Lock()
+	globalCompressEnabled := globalCompressConfig.Enabled
+	globalCompressConfigMu.Unlock()
+	if globalCompressEnabled {
+		objectInputs = objectInputs[0:8]
+	}
 	// iterate through the above set of inputs and upload the object.
 	for _, input := range objectInputs {
 		uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false)
@@ -898,6 +904,14 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket
 		{"enc-mp-3", []int64{10499807, 10499963, 7}, mapCopy(metaWithSSEC)},
 	}
 
+	// SSEC can't be used with compression
+	globalCompressConfigMu.Lock()
+	globalCompressEnabled := globalCompressConfig.Enabled
+	globalCompressConfigMu.Unlock()
+	if globalCompressEnabled {
+		objectInputs = objectInputs[0:9]
+	}
+
 	// iterate through the above set of inputs and upload the object.
 	for _, input := range objectInputs {
 		uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false)
@@ -2119,7 +2133,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 // Wrapper for calling Copy Object API handler tests for both Erasure multiple disks and single node setup.
 func TestAPICopyObjectHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject", "PutObject"})
 }
 
 func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -2150,7 +2164,10 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		new(bytes.Buffer),
 		new(bytes.Buffer),
 	}
-
+	bucketInfo, err := obj.GetBucketInfo(context.Background(), bucketName, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Test -1: %s: Failed to get bucket info: <ERROR> %s", instanceType, err)
+	}
 	// set of inputs for uploading the objects before tests for downloading is done.
 	putObjectInputs := []struct {
 		bucketName    string
@@ -2167,21 +2184,29 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// used for anonymous HTTP request test.
 		{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
 	}
-
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
-		// uploading the object.
-		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewReader(input.textData), input.contentLength, input.md5sum, ""), ObjectOptions{UserDefined: input.metaData})
-		// if object upload fails stop the test.
+		rec := httptest.NewRecorder()
+		req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", input.bucketName, input.objectName),
+			input.contentLength, bytes.NewReader(input.textData), credentials.AccessKey, credentials.SecretKey, nil)
 		if err != nil {
-			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
+			t.Fatalf("Test %d: Failed to create HTTP request for Put Object: <ERROR> %v", i, err)
 		}
-		if objInfo.ETag != input.md5sum {
-			t.Fatalf("Put Object case %d:  Checksum mismatched: <ERROR> got %s, expected %s", i+1, input.md5sum, objInfo.ETag)
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != http.StatusOK {
+			b, err := io.ReadAll(rec.Body)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var apiErr APIErrorResponse
+			err = xml.Unmarshal(b, &apiErr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotErr := apiErr.Code
+			t.Errorf("test %d: want api got %q", i, gotErr)
 		}
 	}
-
 	// test cases with inputs and expected result for Copy Object.
 	testCases := []struct {
 		bucketName           string
@@ -2434,6 +2459,22 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	}
 
 	for i, testCase := range testCases {
+		if bucketInfo.Versioning {
+			if strings.Contains(testCase.copySourceHeader, "versionId=null") {
+				testCase.expectedRespStatus = http.StatusNotFound
+			}
+		}
+		values := url.Values{}
+		if testCase.expectedRespStatus == http.StatusOK {
+			r, err := obj.GetObjectNInfo(context.Background(), testCase.bucketName, objectName, nil, nil, opts)
+			if err != nil {
+				t.Fatalf("Test %d: %s reading completed file failed: <ERROR> %v", i, instanceType, err)
+			}
+			r.Close()
+			if r.ObjInfo.VersionID != "" {
+				values.Set(xhttp.VersionID, r.ObjInfo.VersionID)
+			}
+		}
 		var req *http.Request
 		var reqV2 *http.Request
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
@@ -2446,7 +2487,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		}
 		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
 		if testCase.copySourceHeader != "" {
-			req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
+			if values.Encode() != "" && !strings.Contains(testCase.copySourceHeader, "?") {
+				req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader+"?"+values.Encode())
+			} else {
+				req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
+			}
 		}
 		if testCase.copyModifiedHeader != "" {
 			req.Header.Set("X-Amz-Copy-Source-If-Modified-Since", testCase.copyModifiedHeader)
@@ -2552,10 +2597,10 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			if testCase.copySourceSame {
 				// encryption will rotate creds, so fail only for non-encryption scenario.
 				if GlobalKMS == nil {
-					t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code)
+					t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, recV2.Code)
 				}
 			} else {
-				t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+				t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
 			}
 		}
 	}
@@ -2588,7 +2633,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 // The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it.
 func TestAPINewMultipartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPINewMultipartHandler, []string{"NewMultipart"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPINewMultipartHandler, endpoints: []string{"NewMultipart"}})
 }
 
 func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -2727,7 +2772,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 // The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it.
 func TestAPINewMultipartHandlerParallel(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPINewMultipartHandlerParallel, []string{"NewMultipart"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPINewMultipartHandlerParallel, endpoints: []string{"NewMultipart"}})
 }
 
 func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -2790,7 +2835,7 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
 // The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it.
 func TestAPICompleteMultipartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICompleteMultipartHandler, []string{"CompleteMultipart"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICompleteMultipartHandler, endpoints: []string{"CompleteMultipart"}})
 }
 
 func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -2914,7 +2959,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	s3MD5 := getCompleteMultipartMD5(inputParts[3].parts)
 
 	// generating the response body content for the success case.
-	successResponse := generateCompleteMultpartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5})
+	successResponse := generateCompleteMultipartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5}, nil)
 	encodedSuccessResponse := encodeResponse(successResponse)
 
 	ctx := context.Background()
@@ -3159,7 +3204,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 // The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it.
 func TestAPIAbortMultipartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIAbortMultipartHandler, []string{"AbortMultipart"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIAbortMultipartHandler, endpoints: []string{"AbortMultipart"}})
 }
 
 func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -3319,7 +3364,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 // Wrapper for calling Delete Object API handler tests for both Erasure multiple disks and FS single drive setup.
 func TestAPIDeleteObjectHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIDeleteObjectHandler, []string{"DeleteObject"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIDeleteObjectHandler, endpoints: []string{"DeleteObject"}})
 }
 
 func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -3881,8 +3926,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 //	when signature type of the HTTP request is `Presigned`.
 func TestAPIListObjectPartsHandlerPreSign(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIListObjectPartsHandlerPreSign,
-		[]string{"PutObjectPart", "NewMultipart", "ListObjectParts"})
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIListObjectPartsHandlerPreSign, endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"}})
 }
 
 func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,

cmd/object-multipart-handlers.go

@@ -116,11 +116,6 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 			return
 		}
 
-		if crypto.SSEC.IsRequested(r.Header) && isCompressible(r.Header, object) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionWithCompression), r.URL)
-			return
-		}
-
 		_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
 		ssecRepHeaders := []string{
 			"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
@@ -773,6 +768,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
+	_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
 	var objectEncryptionKey crypto.ObjectKey
 	if isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) && !replicationStatus {
@@ -795,7 +791,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			}
 		}
 
-		_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
 		if !(sourceReplReq && crypto.SSEC.IsEncrypted(mi.UserDefined)) {
 			// Calculating object encryption key
 			key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined)
@@ -847,6 +842,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 	opts.IndexCB = idxCb
 
+	opts.ReplicationRequest = sourceReplReq
 	putObjectPart := objectAPI.PutObjectPart
 
 	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, pReader, opts)
@@ -1028,7 +1024,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		}
 	}
 
-	setPutObjHeaders(w, objInfo, false)
+	setPutObjHeaders(w, objInfo, false, r.Header)
 	if dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
 		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 	}
@@ -1040,7 +1036,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	// Get object location.
 	location := getObjectLocation(r, globalDomainNames, bucket, object)
 	// Generate complete multipart response.
-	response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo)
+	response := generateCompleteMultipartUploadResponse(bucket, object, location, objInfo, r.Header)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// Write success response.

cmd/object_api_suite_test.go

@@ -511,12 +511,12 @@ func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHand
 	}
 }
 
-func enableCompression(t *testing.T, encrypt bool) {
+func enableCompression(t *testing.T, encrypt bool, mimeTypes []string, extensions []string) {
 	// Enable compression and exec...
 	globalCompressConfigMu.Lock()
 	globalCompressConfig.Enabled = true
-	globalCompressConfig.MimeTypes = nil
-	globalCompressConfig.Extensions = nil
+	globalCompressConfig.MimeTypes = mimeTypes
+	globalCompressConfig.Extensions = extensions
 	globalCompressConfig.AllowEncrypted = encrypt
 	globalCompressConfigMu.Unlock()
 	if encrypt {
@@ -553,11 +553,14 @@ func resetCompressEncryption() {
 	GlobalKMS = nil
 }
 
-func execExtended(t *testing.T, fn func(t *testing.T)) {
+func execExtended(t *testing.T, fn func(t *testing.T, init func(), bucketOptions MakeBucketOptions)) {
 	// Exec with default settings...
 	resetCompressEncryption()
 	t.Run("default", func(t *testing.T) {
-		fn(t)
+		fn(t, nil, MakeBucketOptions{})
+	})
+	t.Run("defaultVerioned", func(t *testing.T) {
+		fn(t, nil, MakeBucketOptions{VersioningEnabled: true})
 	})
 
 	if testing.Short() {
@@ -565,28 +568,55 @@ func execExtended(t *testing.T, fn func(t *testing.T)) {
 	}
 
 	t.Run("compressed", func(t *testing.T) {
-		resetCompressEncryption()
-		enableCompression(t, false)
-		fn(t)
+		fn(t, func() {
+			resetCompressEncryption()
+			enableCompression(t, false, []string{"*"}, []string{"*"})
+		}, MakeBucketOptions{})
+	})
+	t.Run("compressedVerioned", func(t *testing.T) {
+		fn(t, func() {
+			resetCompressEncryption()
+			enableCompression(t, false, []string{"*"}, []string{"*"})
+		}, MakeBucketOptions{
+			VersioningEnabled: true,
+		})
 	})
 
 	t.Run("encrypted", func(t *testing.T) {
-		resetCompressEncryption()
-		enableEncryption(t)
-		fn(t)
+		fn(t, func() {
+			resetCompressEncryption()
+			enableEncryption(t)
+		}, MakeBucketOptions{})
+	})
+	t.Run("encryptedVerioned", func(t *testing.T) {
+		fn(t, func() {
+			resetCompressEncryption()
+			enableEncryption(t)
+		}, MakeBucketOptions{
+			VersioningEnabled: true,
+		})
 	})
 
 	t.Run("compressed+encrypted", func(t *testing.T) {
-		resetCompressEncryption()
-		enableCompression(t, true)
-		fn(t)
+		fn(t, func() {
+			resetCompressEncryption()
+			enableCompression(t, true, []string{"*"}, []string{"*"})
+		}, MakeBucketOptions{})
+	})
+	t.Run("compressed+encryptedVerioned", func(t *testing.T) {
+		fn(t, func() {
+			resetCompressEncryption()
+			enableCompression(t, true, []string{"*"}, []string{"*"})
+		}, MakeBucketOptions{
+			VersioningEnabled: true,
+		})
 	})
 }
 
 // ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
 // This can be used to test functionality when reading and writing data.
 func ExecExtendedObjectLayerTest(t *testing.T, objTest objTestType) {
-	execExtended(t, func(t *testing.T) {
+	execExtended(t, func(t *testing.T, init func(), bucketOptions MakeBucketOptions) {
 		ExecObjectLayerTest(t, objTest)
 	})
 }

cmd/s3-zip-handlers.go

@@ -142,7 +142,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 		return
 	}
 
-	zipInfo := zipObjInfo.ArchiveInfo()
+	zipInfo := zipObjInfo.ArchiveInfo(r.Header)
 	if len(zipInfo) == 0 {
 		opts.EncryptFn, err = zipObjInfo.metadataEncryptFn(r.Header)
 		if err != nil {
@@ -233,7 +233,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 }
 
 // listObjectsV2InArchive generates S3 listing result ListObjectsV2Info from zip file, all parameters are already validated by the caller.
-func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, prefix, token, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
+func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, prefix, token, delimiter string, maxKeys int, startAfter string, h http.Header) (ListObjectsV2Info, error) {
 	zipPath, _, err := splitZipExtensionPath(prefix)
 	if err != nil {
 		// Return empty listing
@@ -246,7 +246,7 @@ func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket,
 		return ListObjectsV2Info{}, nil
 	}
 
-	zipInfo := zipObjInfo.ArchiveInfo()
+	zipInfo := zipObjInfo.ArchiveInfo(h)
 	if len(zipInfo) == 0 {
 		// Always update the latest version
 		zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, ObjectOptions{})
@@ -438,7 +438,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
 		return
 	}
 
-	zipInfo := zipObjInfo.ArchiveInfo()
+	zipInfo := zipObjInfo.ArchiveInfo(r.Header)
 	if len(zipInfo) == 0 {
 		opts.EncryptFn, err = zipObjInfo.metadataEncryptFn(r.Header)
 		if err != nil {

cmd/server-main.go

@@ -414,11 +414,12 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 	setGlobalInternodeInterface(ctxt.Interface)
 
 	globalTCPOptions = xhttp.TCPOptions{
-		UserTimeout:    int(ctxt.UserTimeout.Milliseconds()),
-		DriveOPTimeout: globalDriveConfig.GetOPTimeout,
-		Interface:      ctxt.Interface,
-		SendBufSize:    ctxt.SendBufSize,
-		RecvBufSize:    ctxt.RecvBufSize,
+		UserTimeout: int(ctxt.UserTimeout.Milliseconds()),
+		// FIXME: Bring this back when we have valid way to handle deadlines
+		//		DriveOPTimeout: globalDriveConfig.GetOPTimeout,
+		Interface:   ctxt.Interface,
+		SendBufSize: ctxt.SendBufSize,
+		RecvBufSize: ctxt.RecvBufSize,
 	}
 
 	// allow transport to be HTTP/1.1 for proxying.
@@ -430,7 +431,7 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 		RoundTripper: globalRemoteTargetTransport,
 		Logger: func(err error) {
 			if err != nil && !errors.Is(err, context.Canceled) {
-				replLogIf(GlobalContext, err)
+				proxyLogIf(GlobalContext, err)
 			}
 		},
 	})
@@ -834,7 +835,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Set system resources to maximum.
 	bootstrapTrace("setMaxResources", func() {
-		_ = setMaxResources(ctx)
+		_ = setMaxResources(globalServerCtxt)
 	})
 
 	// Verify kernel release and version.

cmd/server-rlimit.go

@@ -22,7 +22,6 @@ import (
 	"runtime/debug"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/cli"
 	"github.com/minio/madmin-go/v3/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/sys"
@@ -45,7 +44,7 @@ func oldLinux() bool {
 	return currentKernel < kernel.Version(4, 0, 0)
 }
 
-func setMaxResources(ctx *cli.Context) (err error) {
+func setMaxResources(ctx serverCtxt) (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, err := sys.GetMaxThreads()
 	if err == nil {
@@ -72,32 +71,28 @@ func setMaxResources(ctx *cli.Context) (err error) {
 		return err
 	}
 
-	// Set max memory limit as current memory limit.
-	if _, maxLimit, err = sys.GetMaxMemoryLimit(); err != nil {
+	_, vssLimit, err := sys.GetMaxMemoryLimit()
+	if err != nil {
 		return err
 	}
 
-	// set debug memory limit instead of GOMEMLIMIT env
-	_ = setDebugMemoryLimit(ctx)
-
-	err = sys.SetMaxMemoryLimit(maxLimit, maxLimit)
-	return err
-}
-
-func setDebugMemoryLimit(ctx *cli.Context) error {
-	if ctx == nil {
-		return nil
+	if vssLimit > 0 && vssLimit < humanize.GiByte {
+		logger.Info("WARNING: maximum virtual memory limit (%s) is too small for 'go runtime', please consider setting `ulimit -v` to unlimited",
+			humanize.IBytes(vssLimit))
 	}
-	if ctx.IsSet("memlimit") {
-		memlimit := ctx.String("memlimit")
-		if memlimit == "" {
-			memlimit = ctx.GlobalString("memlimit")
-		}
-		mlimit, err := humanize.ParseBytes(memlimit)
-		if err != nil {
-			return err
-		}
-		debug.SetMemoryLimit(int64(mlimit))
+
+	if ctx.MemLimit > 0 {
+		maxLimit = ctx.MemLimit
 	}
+
+	if maxLimit > 0 {
+		debug.SetMemoryLimit(int64(maxLimit))
+	}
+
+	// Do not use RLIMIT_AS as that is not useful and at times on systems < 4Gi
+	// this can crash the Go runtime if the value is smaller refer
+	// - https://github.com/golang/go/issues/38010
+	// - https://github.com/golang/go/issues/43699
+	// So do not add `sys.SetMaxMemoryLimit()` this is not useful for any practical purposes.
 	return nil
 }

cmd/sftp-server-driver.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,13 +32,16 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/minio/minio/internal/auth"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/pkg/v3/mimedb"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
 
+// Maximum write offset for incoming SFTP blocks.
+// Set to 100MiB to prevent hostile DOS attacks.
+const ftpMaxWriteOffset = 100 << 20
+
 type sftpDriver struct {
 	permissions *ssh.Permissions
 	endpoint    string
@@ -49,7 +52,7 @@ type sftpMetrics struct{}
 
 var globalSftpMetrics sftpMetrics
 
-func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string, err error) madmin.TraceInfo {
+func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -58,18 +61,25 @@ func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string,
 		TraceType: madmin.TraceFTP,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
-		FuncName:  fmt.Sprintf("sftp USER=%s COMMAND=%s PARAM=%s, Source=%s", user, s.Method, s.Filepath, source),
+		FuncName:  s.Method,
 		Duration:  time.Since(startTime),
 		Path:      s.Filepath,
 		Error:     errStr,
+		Bytes:     sz,
+		Custom: map[string]string{
+			"user":   user,
+			"cmd":    s.Method,
+			"param":  s.Filepath,
+			"source": source,
+		},
 	}
 }
 
-func (m *sftpMetrics) log(s *sftp.Request, user string) func(err error) {
+func (m *sftpMetrics) log(s *sftp.Request, user string) func(sz int64, err error) {
 	startTime := time.Now()
 	source := getSource(2)
-	return func(err error) {
-		globalTrace.Publish(sftpTrace(s, startTime, source, user, err))
+	return func(sz int64, err error) {
+		globalTrace.Publish(sftpTrace(s, startTime, source, user, err, sz))
 	}
 }
 
@@ -90,108 +100,26 @@ func NewSFTPDriver(perms *ssh.Permissions) sftp.Handlers {
 }
 
 func (f *sftpDriver) getMinIOClient() (*minio.Client, error) {
-	ui, ok := globalIAMSys.GetUser(context.Background(), f.AccessKey())
-	if !ok && !globalIAMSys.LDAPConfig.Enabled() {
-		return nil, errNoSuchUser
-	}
-	if !ok && globalIAMSys.LDAPConfig.Enabled() {
-		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), f.AccessKey())
-		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
-			return nil, err
-		}
-		var mcreds *credentials.Credentials
-		if errors.Is(err, errNoSuchServiceAccount) {
-			lookupResult, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(f.AccessKey())
-			if err != nil {
-				return nil, err
-			}
-			expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
-			if err != nil {
-				return nil, err
-			}
-			claims := make(map[string]interface{})
-			claims[expClaim] = UTCNow().Add(expiryDur).Unix()
-			for k, v := range f.permissions.CriticalOptions {
-				claims[k] = v
-			}
-
-			// Set LDAP claims.
-			claims[ldapUserN] = f.AccessKey()
-			claims[ldapUser] = lookupResult.NormDN
-			// Add LDAP attributes that were looked up into the claims.
-			for attribKey, attribValue := range lookupResult.Attributes {
-				claims[ldapAttribPrefix+attribKey] = attribValue
-			}
-
-			cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
-			if err != nil {
-				return nil, err
-			}
-
-			// Set the parent of the temporary access key, this is useful
-			// in obtaining service accounts by this cred.
-			cred.ParentUser = lookupResult.NormDN
-
-			// Set this value to LDAP groups, LDAP user can be part
-			// of large number of groups
-			cred.Groups = targetGroups
-
-			// Set the newly generated credentials, policyName is empty on purpose
-			// LDAP policies are applied automatically using their ldapUser, ldapGroups
-			// mapping.
-			updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
-			if err != nil {
-				return nil, err
-			}
-
-			// Call hook for site replication.
-			replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
-				Type: madmin.SRIAMItemSTSAcc,
-				STSCredential: &madmin.SRSTSCredential{
-					AccessKey:    cred.AccessKey,
-					SecretKey:    cred.SecretKey,
-					SessionToken: cred.SessionToken,
-					ParentUser:   cred.ParentUser,
-				},
-				UpdatedAt: updatedAt,
-			}))
-
-			mcreds = credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken)
-		} else {
-			mcreds = credentials.NewStaticV4(sa.Credentials.AccessKey, sa.Credentials.SecretKey, "")
-		}
-
-		return minio.New(f.endpoint, &minio.Options{
-			Creds:     mcreds,
-			Secure:    globalIsTLS,
-			Transport: globalRemoteFTPClientTransport,
-		})
-	}
-
-	// ok == true - at this point
-
-	if ui.Credentials.IsTemp() {
-		// Temporary credentials are not allowed.
-		return nil, errAuthentication
-	}
-
+	mcreds := credentials.NewStaticV4(
+		f.permissions.CriticalOptions["AccessKey"],
+		f.permissions.CriticalOptions["SecretKey"],
+		f.permissions.CriticalOptions["SessionToken"],
+	)
 	return minio.New(f.endpoint, &minio.Options{
-		Creds:     credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""),
+		Creds:     mcreds,
 		Secure:    globalIsTLS,
 		Transport: globalRemoteFTPClientTransport,
 	})
 }
 
 func (f *sftpDriver) AccessKey() string {
-	if _, ok := f.permissions.CriticalOptions["accessKey"]; !ok {
-		return f.permissions.CriticalOptions[ldapUserN]
-	}
-	return f.permissions.CriticalOptions["accessKey"]
+	return f.permissions.CriticalOptions["AccessKey"]
 }
 
 func (f *sftpDriver) Fileread(r *sftp.Request) (ra io.ReaderAt, err error) {
+	// This is not timing the actual read operation, but the time it takes to prepare the reader.
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	flags := r.Pflags()
 	if !flags.Read {
@@ -269,6 +197,9 @@ func (w *writerAt) WriteAt(b []byte, offset int64) (n int, err error) {
 		n, err = w.w.Write(b)
 		w.nextOffset += int64(n)
 	} else {
+		if offset > w.nextOffset+ftpMaxWriteOffset {
+			return 0, fmt.Errorf("write offset %d is too far ahead of next offset %d", offset, w.nextOffset)
+		}
 		w.buffer[offset] = make([]byte, len(b))
 		copy(w.buffer[offset], b)
 		n = len(b)
@@ -294,7 +225,12 @@ again:
 
 func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer func() {
+		if err != nil {
+			// If there is an error, we never started the goroutine.
+			stopFn(0, err)
+		}
+	}()
 
 	flags := r.Pflags()
 	if !flags.Write {
@@ -329,10 +265,11 @@ func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
 	}
 	wa.wg.Add(1)
 	go func() {
-		_, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{
+		oi, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{
 			ContentType:          mimedb.TypeByExtension(path.Ext(object)),
 			DisableContentSha256: true,
 		})
+		stopFn(oi.Size, err)
 		pr.CloseWithError(err)
 		wa.wg.Done()
 	}()
@@ -341,7 +278,7 @@ func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
 
 func (f *sftpDriver) Filecmd(r *sftp.Request) (err error) {
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	clnt, err := f.getMinIOClient()
 	if err != nil {
@@ -437,7 +374,7 @@ func (f listerAt) ListAt(ls []os.FileInfo, offset int64) (int, error) {
 
 func (f *sftpDriver) Filelist(r *sftp.Request) (la sftp.ListerAt, err error) {
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	clnt, err := f.getMinIOClient()
 	if err != nil {

cmd/sftp-server.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"crypto/subtle"
 	"errors"
@@ -29,18 +28,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/logger"
+	xldap "github.com/minio/pkg/v3/ldap"
 	xsftp "github.com/minio/pkg/v3/sftp"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
 
-type sftpLogger struct{}
-
-func (s *sftpLogger) Info(tag xsftp.LogType, msg string) {
-	logger.Info(msg)
-}
-
 const (
 	kexAlgoDH1SHA1                = "diffie-hellman-group1-sha1"
 	kexAlgoDH14SHA1               = "diffie-hellman-group14-sha1"
@@ -58,6 +54,16 @@ const (
 	tripledescbcID     = "3des-cbc"
 )
 
+var (
+	errSFTPPublicKeyBadFormat = errors.New("the public key provided could not be parsed")
+	errSFTPUserHasNoPolicies  = errors.New("no policies present on this account")
+	errSFTPLDAPNotEnabled     = errors.New("ldap authentication is not enabled")
+)
+
+// if the sftp parameter --trusted-user-ca-key is set, then
+// the final form of the key file will be set as this variable.
+var caPublicKey ssh.PublicKey
+
 // https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=46
 // preferredKexAlgos specifies the default preference for key-exchange
 // algorithms in preference order. The diffie-hellman-group16-sha512 algorithm
@@ -120,6 +126,226 @@ var supportedMACs = []string{
 	"hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1", "hmac-sha1-96",
 }
 
+func sshPubKeyAuth(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
+	return authenticateSSHConnection(c, key, nil)
+}
+
+func sshPasswordAuth(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+	return authenticateSSHConnection(c, nil, pass)
+}
+
+func authenticateSSHConnection(c ssh.ConnMetadata, key ssh.PublicKey, pass []byte) (*ssh.Permissions, error) {
+	user, found := strings.CutSuffix(c.User(), "=ldap")
+	if found {
+		if !globalIAMSys.LDAPConfig.Enabled() {
+			return nil, errSFTPLDAPNotEnabled
+		}
+		return processLDAPAuthentication(key, pass, user)
+	}
+
+	user, found = strings.CutSuffix(c.User(), "=svc")
+	if found {
+		goto internalAuth
+	}
+
+	if globalIAMSys.LDAPConfig.Enabled() {
+		perms, _ := processLDAPAuthentication(key, pass, user)
+		if perms != nil {
+			return perms, nil
+		}
+	}
+
+internalAuth:
+	ui, ok := globalIAMSys.GetUser(context.Background(), user)
+	if !ok {
+		return nil, errNoSuchUser
+	}
+
+	if caPublicKey != nil {
+		err := validateKey(c, key)
+		if err != nil {
+			return nil, errAuthentication
+		}
+	} else {
+
+		// Temporary credentials are not allowed.
+		if ui.Credentials.IsTemp() {
+			return nil, errAuthentication
+		}
+
+		if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) != 1 {
+			return nil, errAuthentication
+		}
+	}
+
+	return &ssh.Permissions{
+		CriticalOptions: map[string]string{
+			"AccessKey":    ui.Credentials.AccessKey,
+			"SecretKey":    ui.Credentials.SecretKey,
+			"SessionToken": ui.Credentials.SessionToken,
+		},
+		Extensions: make(map[string]string),
+	}, nil
+}
+
+func processLDAPAuthentication(key ssh.PublicKey, pass []byte, user string) (perms *ssh.Permissions, err error) {
+	var lookupResult *xldap.DNSearchResult
+	var targetGroups []string
+
+	if pass == nil && key == nil {
+		return nil, errAuthentication
+	}
+
+	if pass != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), user)
+		if err == nil {
+			if subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), pass) != 1 {
+				return nil, errAuthentication
+			}
+
+			return &ssh.Permissions{
+				CriticalOptions: map[string]string{
+					"AccessKey":    sa.Credentials.AccessKey,
+					"SecretKey":    sa.Credentials.SecretKey,
+					"SessionToken": sa.Credentials.SessionToken,
+				},
+				Extensions: make(map[string]string),
+			}, nil
+		}
+
+		if !errors.Is(err, errNoSuchServiceAccount) {
+			return nil, err
+		}
+
+		lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.Bind(user, string(pass))
+		if err != nil {
+			return nil, err
+		}
+
+	} else if key != nil {
+
+		lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(user)
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+	if lookupResult == nil {
+		return nil, errNoSuchUser
+	}
+
+	ldapPolicies, _ := globalIAMSys.PolicyDBGet(lookupResult.NormDN, targetGroups...)
+	if len(ldapPolicies) == 0 {
+		return nil, errSFTPUserHasNoPolicies
+	}
+
+	claims := make(map[string]interface{})
+	for attribKey, attribValue := range lookupResult.Attributes {
+		// we skip multi-value attributes here, as they cannot
+		// be stored in the critical options.
+		if len(attribValue) != 1 {
+			continue
+		}
+
+		if attribKey == "sshPublicKey" && key != nil {
+			key2, _, _, _, err := ssh.ParseAuthorizedKey([]byte(attribValue[0]))
+			if err != nil {
+				return nil, errSFTPPublicKeyBadFormat
+			}
+
+			if subtle.ConstantTimeCompare(key2.Marshal(), key.Marshal()) != 1 {
+				return nil, errAuthentication
+			}
+		}
+		claims[ldapAttribPrefix+attribKey] = attribValue[0]
+	}
+
+	expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
+	if err != nil {
+		return nil, err
+	}
+
+	claims[expClaim] = UTCNow().Add(expiryDur).Unix()
+	claims[ldapUserN] = user
+	claims[ldapUser] = lookupResult.NormDN
+
+	cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set the parent of the temporary access key, this is useful
+	// in obtaining service accounts by this cred.
+	cred.ParentUser = lookupResult.NormDN
+
+	// Set this value to LDAP groups, LDAP user can be part
+	// of large number of groups
+	cred.Groups = targetGroups
+
+	// Set the newly generated credentials, policyName is empty on purpose
+	// LDAP policies are applied automatically using their ldapUser, ldapGroups
+	// mapping.
+	updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
+	if err != nil {
+		return nil, err
+	}
+
+	replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+		Type: madmin.SRIAMItemSTSAcc,
+		STSCredential: &madmin.SRSTSCredential{
+			AccessKey:    cred.AccessKey,
+			SecretKey:    cred.SecretKey,
+			SessionToken: cred.SessionToken,
+			ParentUser:   cred.ParentUser,
+		},
+		UpdatedAt: updatedAt,
+	}))
+
+	return &ssh.Permissions{
+		CriticalOptions: map[string]string{
+			"AccessKey":    cred.AccessKey,
+			"SecretKey":    cred.SecretKey,
+			"SessionToken": cred.SessionToken,
+		},
+		Extensions: make(map[string]string),
+	}, nil
+}
+
+func validateKey(c ssh.ConnMetadata, clientKey ssh.PublicKey) (err error) {
+	if caPublicKey == nil {
+		return errors.New("public key authority validation requested but no ca public key specified.")
+	}
+
+	cert, ok := clientKey.(*ssh.Certificate)
+	if !ok {
+		return errSftpPublicKeyWithoutCert
+	}
+
+	// ssh.CheckCert called by ssh.Authenticate accepts certificates
+	// with empty principles list so we block those in here.
+	if len(cert.ValidPrincipals) == 0 {
+		return errSftpCertWithoutPrincipals
+	}
+
+	// Verify that certificate provided by user is issued by trusted CA,
+	// username in authentication request matches to identities in certificate
+	// and that certificate type is correct.
+	checker := ssh.CertChecker{}
+	checker.IsUserAuthority = func(k ssh.PublicKey) bool {
+		return subtle.ConstantTimeCompare(caPublicKey.Marshal(), k.Marshal()) == 1
+	}
+
+	_, err = checker.Authenticate(c, clientKey)
+	return
+}
+
+type sftpLogger struct{}
+
+func (s *sftpLogger) Info(tag xsftp.LogType, msg string) {
+	logger.Info(msg)
+}
+
 func (s *sftpLogger) Error(tag xsftp.LogType, err error) {
 	switch tag {
 	case xsftp.AcceptNetworkError:
@@ -160,16 +386,19 @@ func filterAlgos(arg string, want []string, allowed []string) []string {
 
 func startSFTPServer(args []string) {
 	var (
-		port          int
-		publicIP      string
-		sshPrivateKey string
-		userCaKeyFile string
+		port            int
+		publicIP        string
+		sshPrivateKey   string
+		userCaKeyFile   string
+		disablePassAuth bool
 	)
+
 	allowPubKeys := supportedPubKeyAuthAlgos
 	allowKexAlgos := preferredKexAlgos
 	allowCiphers := preferredCiphers
 	allowMACs := supportedMACs
 	var err error
+
 	for _, arg := range args {
 		tokens := strings.SplitN(arg, "=", 2)
 		if len(tokens) != 2 {
@@ -201,6 +430,8 @@ func startSFTPServer(args []string) {
 			allowMACs = filterAlgos(arg, strings.Split(tokens[1], ","), supportedMACs)
 		case "trusted-user-ca-key":
 			userCaKeyFile = tokens[1]
+		case "password-auth":
+			disablePassAuth, _ = strconv.ParseBool(tokens[1])
 		}
 	}
 
@@ -222,6 +453,18 @@ func startSFTPServer(args []string) {
 		logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not parseable: %v", err), "unable to start SFTP server")
 	}
 
+	if userCaKeyFile != "" {
+		keyBytes, err := os.ReadFile(userCaKeyFile)
+		if err != nil {
+			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not accessible: %v", err), "unable to start SFTP server")
+		}
+
+		caPublicKey, _, _, _, err = ssh.ParseAuthorizedKey(keyBytes)
+		if err != nil {
+			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not parseable: %v", err), "unable to start SFTP server")
+		}
+	}
+
 	// An SSH server is represented by a ServerConfig, which holds
 	// certificate details and handles authentication of ServerConns.
 	sshConfig := &ssh.ServerConfig{
@@ -231,116 +474,13 @@ func startSFTPServer(args []string) {
 			MACs:         allowMACs,
 		},
 		PublicKeyAuthAlgorithms: allowPubKeys,
-		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
-			if globalIAMSys.LDAPConfig.Enabled() {
-				sa, _, err := globalIAMSys.getServiceAccount(context.Background(), c.User())
-				if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
-					return nil, err
-				}
-				if errors.Is(err, errNoSuchServiceAccount) {
-					lookupResult, targetGroups, err := globalIAMSys.LDAPConfig.Bind(c.User(), string(pass))
-					if err != nil {
-						return nil, err
-					}
-					targetUser := lookupResult.NormDN
-					ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
-					if len(ldapPolicies) == 0 {
-						return nil, errAuthentication
-					}
-					criticalOptions := map[string]string{
-						ldapUser:       targetUser,
-						ldapActualUser: lookupResult.ActualDN,
-						ldapUserN:      c.User(),
-					}
-					for attribKey, attribValue := range lookupResult.Attributes {
-						// we skip multi-value attributes here, as they cannot
-						// be stored in the critical options.
-						if len(attribValue) == 1 {
-							criticalOptions[ldapAttribPrefix+attribKey] = attribValue[0]
-						}
-					}
-
-					return &ssh.Permissions{
-						CriticalOptions: criticalOptions,
-						Extensions:      make(map[string]string),
-					}, nil
-				}
-				if subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), pass) == 1 {
-					return &ssh.Permissions{
-						CriticalOptions: map[string]string{
-							"accessKey": c.User(),
-						},
-						Extensions: make(map[string]string),
-					}, nil
-				}
-				return nil, errAuthentication
-			}
-
-			ui, ok := globalIAMSys.GetUser(context.Background(), c.User())
-			if !ok {
-				return nil, errNoSuchUser
-			}
-
-			if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) == 1 {
-				return &ssh.Permissions{
-					CriticalOptions: map[string]string{
-						"accessKey": c.User(),
-					},
-					Extensions: make(map[string]string),
-				}, nil
-			}
-			return nil, errAuthentication
-		},
+		PublicKeyCallback:       sshPubKeyAuth,
 	}
 
-	if userCaKeyFile != "" {
-		keyBytes, err := os.ReadFile(userCaKeyFile)
-		if err != nil {
-			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not accessible: %v", err), "unable to start SFTP server")
-		}
-
-		caPublicKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes)
-		if err != nil {
-			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not parseable: %v", err), "unable to start SFTP server")
-		}
-
-		sshConfig.PublicKeyCallback = func(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
-			_, ok := globalIAMSys.GetUser(context.Background(), c.User())
-			if !ok {
-				return nil, errNoSuchUser
-			}
-
-			// Verify that client provided certificate, not only public key.
-			cert, ok := key.(*ssh.Certificate)
-			if !ok {
-				return nil, errSftpPublicKeyWithoutCert
-			}
-
-			// ssh.CheckCert called by ssh.Authenticate accepts certificates
-			// with empty principles list so we block those in here.
-			if len(cert.ValidPrincipals) == 0 {
-				return nil, errSftpCertWithoutPrincipals
-			}
-
-			// Verify that certificate provided by user is issued by trusted CA,
-			// username in authentication request matches to identities in certificate
-			// and that certificate type is correct.
-			checker := ssh.CertChecker{}
-			checker.IsUserAuthority = func(k ssh.PublicKey) bool {
-				return bytes.Equal(k.Marshal(), caPublicKey.Marshal())
-			}
-			_, err = checker.Authenticate(c, key)
-			if err != nil {
-				return nil, err
-			}
-
-			return &ssh.Permissions{
-				CriticalOptions: map[string]string{
-					"accessKey": c.User(),
-				},
-				Extensions: make(map[string]string),
-			}, nil
-		}
+	if !disablePassAuth {
+		sshConfig.PasswordCallback = sshPasswordAuth
+	} else {
+		sshConfig.PasswordCallback = nil
 	}
 
 	sshConfig.AddHostKey(private)

cmd/sftp-server_test.go

@@ -0,0 +1,289 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/minio/madmin-go/v3"
+	"golang.org/x/crypto/ssh"
+)
+
+type MockConnMeta struct {
+	username string
+}
+
+func (m *MockConnMeta) User() string {
+	return m.username
+}
+
+func (m *MockConnMeta) SessionID() []byte {
+	return []byte{}
+}
+
+func (m *MockConnMeta) ClientVersion() []byte {
+	return []byte{}
+}
+
+func (m *MockConnMeta) ServerVersion() []byte {
+	return []byte{}
+}
+
+func (m *MockConnMeta) RemoteAddr() net.Addr {
+	return nil
+}
+
+func (m *MockConnMeta) LocalAddr() net.Addr {
+	return nil
+}
+
+func newSSHConnMock(username string) ssh.ConnMetadata {
+	return &MockConnMeta{username: username}
+}
+
+func TestSFTPAuthentication(t *testing.T) {
+	for i, testCase := range iamTestSuites {
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
+			func(t *testing.T) {
+				c := &check{t, testCase.serverType}
+				suite := testCase
+
+				suite.SetUpSuite(c)
+
+				suite.SFTPServiceAccountLogin(c)
+				suite.SFTPInvalidServiceAccountPassword(c)
+
+				// LDAP tests
+				ldapServer := os.Getenv(EnvTestLDAPServer)
+				if ldapServer == "" {
+					c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer)
+				}
+
+				suite.SetUpLDAP(c, ldapServer)
+
+				suite.SFTPFailedAuthDueToMissingPolicy(c)
+				suite.SFTPFailedAuthDueToInvalidUser(c)
+				suite.SFTPFailedForcedServiceAccountAuthOnLDAPUser(c)
+				suite.SFTPFailedAuthDueToInvalidPassword(c)
+
+				suite.SFTPValidLDAPLoginWithPassword(c)
+
+				suite.SFTPPublicKeyAuthentication(c)
+				suite.SFTPFailedPublicKeyAuthenticationInvalidKey(c)
+
+				suite.TearDownSuite(c)
+			},
+		)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedPublicKeyAuthenticationInvalidKey(c *check) {
+	keyBytes, err := os.ReadFile("./testdata/invalid_test_key.pub")
+	if err != nil {
+		c.Fatalf("could not read test key file: %s", err)
+	}
+
+	testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes)
+	if err != nil {
+		c.Fatalf("could not parse test key file: %s", err)
+	}
+
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err == nil || !errors.Is(err, errAuthentication) {
+		c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPPublicKeyAuthentication(c *check) {
+	keyBytes, err := os.ReadFile("./testdata/dillon_test_key.pub")
+	if err != nil {
+		c.Fatalf("could not read test key file: %s", err)
+	}
+
+	testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes)
+	if err != nil {
+		c.Fatalf("could not parse test key file: %s", err)
+	}
+
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err != nil {
+		c.Fatalf("expected no error but got(%s)", err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err != nil {
+		c.Fatalf("expected no error but got(%s)", err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedAuthDueToMissingPolicy(c *check) {
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err == nil || !errors.Is(err, errSFTPUserHasNoPolicies) {
+		c.Fatalf("expected err(%s) but got (%s)", errSFTPUserHasNoPolicies, err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedAuthDueToInvalidUser(c *check) {
+	newSSHCon := newSSHConnMock("dillon_error")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon_error"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedForcedServiceAccountAuthOnLDAPUser(c *check) {
+	newSSHCon := newSSHConnMock("dillon=svc")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedAuthDueToInvalidPassword(c *check) {
+	newSSHCon := newSSHConnMock("dillon")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon_error"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPInvalidServiceAccountPassword(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	if err != nil {
+		c.Fatalf("unable to set policy: %v", err)
+	}
+
+	newSSHCon := newSSHConnMock(accessKey + "=svc")
+	_, err = sshPasswordAuth(newSSHCon, []byte("invalid"))
+	if err == nil || !errors.Is(err, errAuthentication) {
+		c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err)
+	}
+
+	newSSHCon = newSSHConnMock(accessKey)
+	_, err = sshPasswordAuth(newSSHCon, []byte("invalid"))
+	if err == nil || !errors.Is(err, errAuthentication) {
+		c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPServiceAccountLogin(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	if err != nil {
+		c.Fatalf("unable to set policy: %v", err)
+	}
+
+	newSSHCon := newSSHConnMock(accessKey + "=svc")
+	_, err = sshPasswordAuth(newSSHCon, []byte(secretKey))
+	if err != nil {
+		c.Fatalf("expected no error but got (%s)", err)
+	}
+
+	newSSHCon = newSSHConnMock(accessKey)
+	_, err = sshPasswordAuth(newSSHCon, []byte(secretKey))
+	if err != nil {
+		c.Fatalf("expected no error but got (%s)", err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPValidLDAPLoginWithPassword(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	// we need to do this so that the user has a policy before authentication.
+	// ldap user accounts without policies are denied access in sftp.
+	policy := "mypolicy"
+	policyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::BUCKET/*"
+   ]
+  }
+ ]
+}`)
+
+	err := s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
+	err = s.adm.SetPolicy(ctx, policy, userDN, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err = sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err != nil {
+		c.Fatal("Password authentication failed for user (dillon):", err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err != nil {
+		c.Fatal("Password authentication failed for user (dillon):", err)
+	}
+}

cmd/signature-v4-utils.go

@@ -152,11 +152,14 @@ func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, A
 			// Check if server has initialized, then only proceed
 			// to check for IAM users otherwise its okay for clients
 			// to retry with 503 errors when server is coming up.
-			return auth.Credentials{}, false, ErrServerNotInitialized
+			return auth.Credentials{}, false, ErrIAMNotInitialized
 		}
 
 		// Check if the access key is part of users credentials.
-		u, ok := globalIAMSys.GetUser(r.Context(), accessKey)
+		u, ok, err := globalIAMSys.CheckKey(r.Context(), accessKey)
+		if err != nil {
+			return auth.Credentials{}, false, ErrIAMNotInitialized
+		}
 		if !ok {
 			// Credentials could be valid but disabled - return a different
 			// error in such a scenario.

cmd/signature-v4_test.go

@@ -37,6 +37,12 @@ func niceError(code APIErrorCode) string {
 }
 
 func TestDoesPolicySignatureMatch(t *testing.T) {
+	_, fsDir, err := prepareFS(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeRoots([]string{fsDir})
+
 	credentialTemplate := "%s/%s/%s/s3/aws4_request"
 	now := UTCNow()
 	accessKey := globalActiveCred.AccessKey

cmd/storage-datatypes.go

@@ -502,6 +502,23 @@ type RenameDataResp struct {
 	OldDataDir string // contains '<uuid>', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir))
 }
 
+const (
+	checkPartUnknown int = iota
+
+	// Changing the order can cause a data loss
+	// when running two nodes with incompatible versions
+	checkPartSuccess
+	checkPartDiskNotFound
+	checkPartVolumeNotFound
+	checkPartFileNotFound
+	checkPartFileCorrupt
+)
+
+// CheckPartsResp is a response of the storage CheckParts and VerifyFile APIs
+type CheckPartsResp struct {
+	Results []int
+}
+
 // LocalDiskIDs - GetLocalIDs response.
 type LocalDiskIDs struct {
 	IDs []string

cmd/storage-datatypes_gen.go

@@ -273,6 +273,145 @@ func (z *CheckPartsHandlerParams) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *CheckPartsResp) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Results":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Results")
+				return
+			}
+			if cap(z.Results) >= int(zb0002) {
+				z.Results = (z.Results)[:zb0002]
+			} else {
+				z.Results = make([]int, zb0002)
+			}
+			for za0001 := range z.Results {
+				z.Results[za0001], err = dc.ReadInt()
+				if err != nil {
+					err = msgp.WrapError(err, "Results", za0001)
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *CheckPartsResp) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "Results"
+	err = en.Append(0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Results)))
+	if err != nil {
+		err = msgp.WrapError(err, "Results")
+		return
+	}
+	for za0001 := range z.Results {
+		err = en.WriteInt(z.Results[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "Results", za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *CheckPartsResp) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "Results"
+	o = append(o, 0x81, 0xa7, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Results)))
+	for za0001 := range z.Results {
+		o = msgp.AppendInt(o, z.Results[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *CheckPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Results":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Results")
+				return
+			}
+			if cap(z.Results) >= int(zb0002) {
+				z.Results = (z.Results)[:zb0002]
+			} else {
+				z.Results = make([]int, zb0002)
+			}
+			for za0001 := range z.Results {
+				z.Results[za0001], bts, err = msgp.ReadIntBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Results", za0001)
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *CheckPartsResp) Msgsize() (s int) {
+	s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.Results) * (msgp.IntSize))
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *DeleteFileHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/storage-datatypes_gen_test.go

@@ -235,6 +235,119 @@ func BenchmarkDecodeCheckPartsHandlerParams(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalCheckPartsResp(t *testing.T) {
+	v := CheckPartsResp{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgCheckPartsResp(b *testing.B) {
+	v := CheckPartsResp{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgCheckPartsResp(b *testing.B) {
+	v := CheckPartsResp{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalCheckPartsResp(b *testing.B) {
+	v := CheckPartsResp{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeCheckPartsResp(t *testing.T) {
+	v := CheckPartsResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeCheckPartsResp Msgsize() is inaccurate")
+	}
+
+	vn := CheckPartsResp{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeCheckPartsResp(b *testing.B) {
+	v := CheckPartsResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeCheckPartsResp(b *testing.B) {
+	v := CheckPartsResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalDeleteFileHandlerParams(t *testing.T) {
 	v := DeleteFileHandlerParams{}
 	bts, err := v.MarshalMsg(nil)

cmd/storage-interface.go

@@ -94,9 +94,9 @@ type StorageAPI interface {
 	CreateFile(ctx context.Context, origvolume, olume, path string, size int64, reader io.Reader) error
 	ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error)
 	RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
-	CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
+	CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error)
 	Delete(ctx context.Context, volume string, path string, opts DeleteOptions) (err error)
-	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
+	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error)
 	StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
 	ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error
 	CleanAbandonedData(ctx context.Context, volume string, path string) error

cmd/storage-rest-client.go

@@ -449,14 +449,18 @@ func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, pa
 }
 
 // CheckParts - stat all file parts.
-func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
-	_, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{
+func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
+	var resp *CheckPartsResp
+	resp, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{
 		DiskID:   *client.diskID.Load(),
 		Volume:   volume,
 		FilePath: path,
 		FI:       fi,
 	})
-	return toStorageErr(err)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
 }
 
 // RenameData - rename source path to destination path atomically, metadata and data file.
@@ -748,33 +752,33 @@ func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcP
 	return toStorageErr(err)
 }
 
-func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
+func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 
 	var reader bytes.Buffer
 	if err := msgp.Encode(&reader, &fi); err != nil {
-		return err
+		return nil, err
 	}
 
 	respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1)
 	defer xhttp.DrainBody(respBody)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	respReader, err := waitForHTTPResponse(respBody)
 	if err != nil {
-		return toStorageErr(err)
+		return nil, toStorageErr(err)
 	}
 
-	verifyResp := &VerifyFileResp{}
+	verifyResp := &CheckPartsResp{}
 	if err = gob.NewDecoder(respReader).Decode(verifyResp); err != nil {
-		return toStorageErr(err)
+		return nil, toStorageErr(err)
 	}
 
-	return toStorageErr(verifyResp.Err)
+	return verifyResp, nil
 }
 
 func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {

cmd/storage-rest-common.go

@@ -20,7 +20,7 @@ package cmd
 //go:generate msgp -file $GOFILE -unexported
 
 const (
-	storageRESTVersion       = "v57" // Remove TotalTokens from DiskMetrics
+	storageRESTVersion       = "v58" // Change VerifyFile signature
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )

cmd/storage-rest-server.go

@@ -58,7 +58,7 @@ type storageRESTServer struct {
 }
 
 var (
-	storageCheckPartsRPC       = grid.NewSingleHandler[*CheckPartsHandlerParams, grid.NoPayload](grid.HandlerCheckParts, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, grid.NewNoPayload)
+	storageCheckPartsRPC       = grid.NewSingleHandler[*CheckPartsHandlerParams, *CheckPartsResp](grid.HandlerCheckParts2, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, func() *CheckPartsResp { return &CheckPartsResp{} })
 	storageDeleteFileRPC       = grid.NewSingleHandler[*DeleteFileHandlerParams, grid.NoPayload](grid.HandlerDeleteFile, func() *DeleteFileHandlerParams { return &DeleteFileHandlerParams{} }, grid.NewNoPayload).AllowCallRequestPool(true)
 	storageDeleteVersionRPC    = grid.NewSingleHandler[*DeleteVersionHandlerParams, grid.NoPayload](grid.HandlerDeleteVersion, func() *DeleteVersionHandlerParams { return &DeleteVersionHandlerParams{} }, grid.NewNoPayload)
 	storageDiskInfoRPC         = grid.NewSingleHandler[*DiskInfoOptions, *DiskInfo](grid.HandlerDiskInfo, func() *DiskInfoOptions { return &DiskInfoOptions{} }, func() *DiskInfo { return &DiskInfo{} }).WithSharedResponse().AllowCallRequestPool(true)
@@ -439,13 +439,15 @@ func (s *storageRESTServer) UpdateMetadataHandler(p *MetadataHandlerParams) (gri
 }
 
 // CheckPartsHandler - check if a file metadata exists.
-func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
+func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (*CheckPartsResp, *grid.RemoteErr) {
 	if !s.checkID(p.DiskID) {
-		return grid.NewNPErr(errDiskNotFound)
+		return nil, grid.NewRemoteErr(errDiskNotFound)
 	}
 	volume := p.Volume
 	filePath := p.FilePath
-	return grid.NewNPErr(s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI))
+
+	resp, err := s.getStorage().CheckParts(context.Background(), volume, filePath, p.FI)
+	return resp, grid.NewRemoteErr(err)
 }
 
 func (s *storageRESTServer) WriteAllHandler(p *WriteAllHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
@@ -1097,11 +1099,6 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
 	}
 }
 
-// VerifyFileResp - VerifyFile()'s response.
-type VerifyFileResp struct {
-	Err error
-}
-
 // VerifyFileHandler - Verify all part of file for bitrot errors.
 func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -1124,13 +1121,15 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
 	setEventStreamHeaders(w)
 	encoder := gob.NewEncoder(w)
 	done := keepHTTPResponseAlive(w)
-	err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi)
+	resp, err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi)
 	done(nil)
-	vresp := &VerifyFileResp{}
+
 	if err != nil {
-		vresp.Err = StorageErr(err.Error())
+		s.writeErrorResponse(w, err)
+		return
 	}
-	encoder.Encode(vresp)
+
+	encoder.Encode(resp)
 }
 
 func checkDiskFatalErrs(errs []error) error {

cmd/sts-handlers.go

@@ -22,9 +22,11 @@ import (
 	"context"
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -82,8 +84,50 @@ const (
 
 	// Role Claim key
 	roleArnClaim = "roleArn"
+
+	// maximum supported STS session policy size
+	maxSTSSessionPolicySize = 2048
 )
 
+type stsClaims map[string]interface{}
+
+func (c stsClaims) populateSessionPolicy(form url.Values) error {
+	if len(form) == 0 {
+		return nil
+	}
+
+	sessionPolicyStr := form.Get(stsPolicy)
+	if len(sessionPolicyStr) == 0 {
+		return nil
+	}
+
+	sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
+	if err != nil {
+		return err
+	}
+
+	// Version in policy must not be empty
+	if sessionPolicy.Version == "" {
+		return errors.New("Version cannot be empty expecting '2012-10-17'")
+	}
+
+	policyBuf, err := json.Marshal(sessionPolicy)
+	if err != nil {
+		return err
+	}
+
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+	// The plain text that you use for both inline and managed session
+	// policies shouldn't exceed maxSTSSessionPolicySize characters.
+	if len(policyBuf) > maxSTSSessionPolicySize {
+		return errSessionPolicyTooLarge
+	}
+
+	c[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
+	return nil
+}
+
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
 type stsAPIHandlers struct{}
 
@@ -212,7 +256,7 @@ func getTokenSigningKey() (string, error) {
 func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRole")
 
-	claims := make(map[string]interface{})
+	claims := stsClaims{}
 	defer logger.AuditLog(ctx, w, r, claims)
 
 	// Check auth here (otherwise r.Form will have unexpected values from
@@ -249,29 +293,11 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	sessionPolicyStr := r.Form.Get(stsPolicy)
-	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
-	// The plain text that you use for both inline and managed session
-	// policies shouldn't exceed 2048 characters.
-	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, errSessionPolicyTooLarge)
+	if err := claims.populateSessionPolicy(r.Form); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
-		if err != nil {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
-			return
-		}
-
-		// Version in policy must not be empty
-		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Version cannot be empty expecting '2012-10-17'"))
-			return
-		}
-	}
-
 	duration, err := openid.GetDefaultExpiration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
@@ -288,10 +314,6 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		claims[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	secret, err := getTokenSigningKey()
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
@@ -342,7 +364,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRoleSSOCommon")
 
-	claims := make(map[string]interface{})
+	claims := stsClaims{}
 	defer logger.AuditLog(ctx, w, r, claims)
 
 	// Parse the incoming form data.
@@ -449,31 +471,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		claims[iamPolicyClaimNameOpenID()] = policyName
 	}
 
-	sessionPolicyStr := r.Form.Get(stsPolicy)
-	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
-	// The plain text that you use for both inline and managed session
-	// policies shouldn't exceed 2048 characters.
-	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
+	if err := claims.populateSessionPolicy(r.Form); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
-		if err != nil {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
-			return
-		}
-
-		// Version in policy must not be empty
-		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid session policy version"))
-			return
-		}
-
-		claims[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	secret, err := getTokenSigningKey()
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
@@ -612,7 +614,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *
 func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRoleWithLDAPIdentity")
 
-	claims := make(map[string]interface{})
+	claims := stsClaims{}
 	defer logger.AuditLog(ctx, w, r, claims, stsLDAPPassword)
 
 	// Parse the incoming form data.
@@ -643,29 +645,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		return
 	}
 
-	sessionPolicyStr := r.Form.Get(stsPolicy)
-	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
-	// The plain text that you use for both inline and managed session
-	// policies shouldn't exceed 2048 characters.
-	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
+	if err := claims.populateSessionPolicy(r.Form); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
-		if err != nil {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
-			return
-		}
-
-		// Version in policy must not be empty
-		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Version needs to be specified in session policy"))
-			return
-		}
-	}
-
 	if !globalIAMSys.Initialized() {
 		writeSTSErrorResponse(ctx, w, ErrSTSIAMNotInitialized, errIAMNotInitialized)
 		return
@@ -708,10 +692,6 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		claims[ldapAttribPrefix+attrib] = value
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		claims[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	secret, err := getTokenSigningKey()
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
@@ -885,7 +865,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	}
 
 	// Associate any service accounts to the certificate CN
-	parentUser := "tls:" + certificate.Subject.CommonName
+	parentUser := "tls" + getKeySeparator() + certificate.Subject.CommonName
 
 	claims[expClaim] = UTCNow().Add(expiry).Unix()
 	claims[subClaim] = certificate.Subject.CommonName
@@ -1010,7 +990,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 		expiry = requestedDuration
 	}
 
-	parentUser := "custom:" + res.Success.User
+	parentUser := "custom" + getKeySeparator() + res.Success.User
 
 	// metadata map
 	claims[expClaim] = UTCNow().Add(time.Duration(expiry) * time.Second).Unix()

cmd/sts-handlers_test.go

@@ -30,10 +30,10 @@ import (
 
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v3"
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
-	ldap "github.com/minio/pkg/v3/ldap"
+	"github.com/minio/pkg/v3/ldap"
 	"golang.org/x/exp/slices"
 )
 

cmd/test-utils_test.go

@@ -106,7 +106,7 @@ func TestMain(m *testing.M) {
 	// logger.AddTarget(console.New())
 
 	// Set system resources to maximum.
-	setMaxResources(nil)
+	setMaxResources(serverCtxt{})
 
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background(), io.Discard)
@@ -1540,7 +1540,7 @@ func removeRoots(roots []string) {
 // initializes the specified API endpoints for the tests.
 // initializes the root and returns its path.
 // return credentials.
-func initAPIHandlerTest(ctx context.Context, obj ObjectLayer, endpoints []string) (string, http.Handler, error) {
+func initAPIHandlerTest(ctx context.Context, obj ObjectLayer, endpoints []string, makeBucketOptions MakeBucketOptions) (string, http.Handler, error) {
 	initAllSubsystems(ctx)
 
 	initConfigSubsystem(ctx, obj)
@@ -1549,9 +1549,8 @@ func initAPIHandlerTest(ctx context.Context, obj ObjectLayer, endpoints []string
 
 	// get random bucket name.
 	bucketName := getRandomBucketName()
-
 	// Create bucket.
-	err := obj.MakeBucket(context.Background(), bucketName, MakeBucketOptions{})
+	err := obj.MakeBucket(context.Background(), bucketName, makeBucketOptions)
 	if err != nil {
 		// failed to create newbucket, return err.
 		return "", nil, err
@@ -1741,9 +1740,17 @@ func ExecObjectLayerAPINilTest(t TestErrHandler, bucketName, objectName, instanc
 	}
 }
 
+type ExecObjectLayerAPITestArgs struct {
+	t                 *testing.T
+	objAPITest        objAPITestType
+	endpoints         []string
+	init              func()
+	makeBucketOptions MakeBucketOptions
+}
+
 // ExecObjectLayerAPITest - executes object layer API tests.
 // Creates single node and Erasure ObjectLayer instance, registers the specified API end points and runs test for both the layers.
-func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
+func ExecObjectLayerAPITest(args ExecObjectLayerAPITestArgs) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -1753,24 +1760,28 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 
 	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
-		t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
+		args.t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
 	}
 
-	bucketFS, fsAPIRouter, err := initAPIHandlerTest(ctx, objLayer, endpoints)
+	bucketFS, fsAPIRouter, err := initAPIHandlerTest(ctx, objLayer, args.endpoints, args.makeBucketOptions)
 	if err != nil {
-		t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
+		args.t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
+	}
+
+	if args.init != nil {
+		args.init()
 	}
 
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		args.t.Fatalf("Unable to initialize server config. %s", err)
 	}
 
 	credentials := globalActiveCred
 
 	// Executing the object layer tests for single node setup.
-	objAPITest(objLayer, ErasureSDStr, bucketFS, fsAPIRouter, credentials, t)
+	args.objAPITest(objLayer, ErasureSDStr, bucketFS, fsAPIRouter, credentials, args.t)
 
 	// reset globals.
 	// this is to make sure that the tests are not affected by modified value.
@@ -1778,23 +1789,27 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 
 	objLayer, erasureDisks, err := prepareErasure16(ctx)
 	if err != nil {
-		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
+		args.t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
 	defer objLayer.Shutdown(ctx)
 
-	bucketErasure, erAPIRouter, err := initAPIHandlerTest(ctx, objLayer, endpoints)
+	bucketErasure, erAPIRouter, err := initAPIHandlerTest(ctx, objLayer, args.endpoints, args.makeBucketOptions)
 	if err != nil {
-		t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
+		args.t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
+	}
+
+	if args.init != nil {
+		args.init()
 	}
 
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatalf("Unable to initialize server config. %s", err)
+		args.t.Fatalf("Unable to initialize server config. %s", err)
 	}
 
 	// Executing the object layer tests for Erasure.
-	objAPITest(objLayer, ErasureTestStr, bucketErasure, erAPIRouter, credentials, t)
+	args.objAPITest(objLayer, ErasureTestStr, bucketErasure, erAPIRouter, credentials, args.t)
 
 	// clean up the temporary test backend.
 	removeRoots(append(erasureDisks, fsDir))
@@ -1803,8 +1818,8 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 // ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
 // This can be used to test functionality when reading and writing data.
 func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
-	execExtended(t, func(t *testing.T) {
-		ExecObjectLayerAPITest(t, objAPITest, endpoints)
+	execExtended(t, func(t *testing.T, init func(), makeBucketOptions MakeBucketOptions) {
+		ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: objAPITest, endpoints: endpoints, init: init, makeBucketOptions: makeBucketOptions})
 	})
 }
 

cmd/testdata/dillon_test_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVGk/SRz4fwTPK0+Ra7WYUGf3o08YkpI0yTMPpHwYoq dillon@example.io

cmd/testdata/invalid_test_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDES4saDDRpoHDVmiYESEQrCYhw8EK7Utj/A/lqxiqZlP6Il3aN2fWu6uJQdWAovZxNeXUf8LIujisW1mJWGZPql0SLKVq6IZ707OAGmKA59IXfF5onRoU9+K4UDL7BJFfix6/3F5OV2WB3ChFrOrXhJ0CZ0sVAfGcV4q72kS19YjZNX3fqCc2HF8UQEaZGKIkw5MtdZI9a1P2bqnPuPGJybRFUzyoQXPge45QT5jnpcsAXOuXcGxbjuqaaHXFNTSKAkCU93TcjAbqUMkTz2mnFz/MnrKJTECN3Fy0GPCCQ5dxmG8p8DyMiNl7JYkX2r3XYgxmioCzkcg8fDs5p0CaQcipu+MA7iK7APKq7v4Zr/wNltXHI3DE9S8J88Hxb2FZAyEhCRfcgGmCVfoZxVNCRHNkGYzfe63BkxtnseUCzpYEhKv02H5u9rjFpdMY37kDfHDVqBbgutdMij+tQAEp1kyqi6TQL+4XHjPHkLaeekW07yB+VI90dK1A9dzTpOvE= liza@example.io

cmd/utils.go

@@ -38,7 +38,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/go-oidc"
+	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/dustin/go-humanize"
 	"github.com/felixge/fgprof"
 	"github.com/minio/madmin-go/v3"

cmd/xl-storage-disk-id-check.go

@@ -288,7 +288,7 @@ func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context, opts DiskInfoOption
 	}
 
 	si := p.updateStorageMetrics(storageMetricDiskInfo)
-	defer si(&err)
+	defer si(0, &err)
 
 	if opts.NoOp {
 		if opts.Metrics {
@@ -341,7 +341,7 @@ func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...strin
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.MakeVolBulk(ctx, volumes...) })
@@ -352,7 +352,7 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.MakeVol(ctx, volume) })
@@ -363,7 +363,7 @@ func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) (vi []VolInfo, err
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.ListVols(ctx)
 }
@@ -373,7 +373,7 @@ func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol
 	if err != nil {
 		return vol, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return xioutil.WithDeadline[VolInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result VolInfo, err error) {
 		return p.storage.StatVol(ctx, volume)
@@ -385,7 +385,7 @@ func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, for
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.DeleteVol(ctx, volume, forceDelete) })
@@ -396,7 +396,7 @@ func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, origvolume, volume,
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.ListDir(ctx, origvolume, volume, dirPath, count)
 }
@@ -407,7 +407,9 @@ func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path
 	if err != nil {
 		return 0, err
 	}
-	defer done(&err)
+	defer func() {
+		done(n, &err)
+	}()
 
 	return xioutil.WithDeadline[int64](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result int64, err error) {
 		return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
@@ -420,7 +422,7 @@ func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, pa
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(int64(len(buf)), &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error {
@@ -433,7 +435,7 @@ func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, origvolume, volum
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(size, &err)
 
 	return p.storage.CreateFile(ctx, origvolume, volume, path, size, io.NopCloser(reader))
 }
@@ -443,7 +445,7 @@ func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(length, &err)
 
 	return xioutil.WithDeadline[io.ReadCloser](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result io.ReadCloser, err error) {
 		return p.storage.ReadFileStream(ctx, volume, path, offset, length)
@@ -455,7 +457,7 @@ func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPat
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) })
@@ -470,7 +472,7 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
 		if err == nil && !skipAccessChecks(dstVolume) {
 			p.storage.setWriteAttribute(p.totalWrites.Add(1))
 		}
-		done(&err)
+		done(0, &err)
 	}()
 
 	// Copy inline data to a new buffer to function with deadlines.
@@ -485,15 +487,16 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
 	})
 }
 
-func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
+func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
 	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
-	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
-	return w.Run(func() error { return p.storage.CheckParts(ctx, volume, path, fi) })
+	return xioutil.WithDeadline[*CheckPartsResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res *CheckPartsResp, err error) {
+		return p.storage.CheckParts(ctx, volume, path, fi)
+	})
 }
 
 func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
@@ -501,7 +504,7 @@ func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path s
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.Delete(ctx, volume, path, deleteOpts) })
@@ -548,7 +551,7 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 				p.storage.setDeleteAttribute(p.totalDeletes.Add(permanentDeletes))
 			}
 		}
-		done(&err)
+		done(0, &err)
 	}()
 
 	errs = p.storage.DeleteVersions(ctx, volume, versions, opts)
@@ -562,12 +565,12 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 	return errs
 }
 
-func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
 	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.VerifyFile(ctx, volume, path, fi)
 }
@@ -577,7 +580,7 @@ func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(int64(len(b)), &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.WriteAll(ctx, volume, path, b) })
@@ -589,7 +592,7 @@ func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path s
 		return err
 	}
 	defer func() {
-		defer done(&err)
+		defer done(0, &err)
 
 		if err == nil && !skipAccessChecks(volume) {
 			if opts.UndoWrite {
@@ -616,7 +619,7 @@ func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.UpdateMetadata(ctx, volume, path, fi, opts) })
@@ -627,7 +630,7 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, vo
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.WriteMetadata(ctx, origvolume, volume, path, fi) })
@@ -638,7 +641,7 @@ func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, origvolume, volu
 	if err != nil {
 		return fi, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return xioutil.WithDeadline[FileInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result FileInfo, err error) {
 		return p.storage.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
@@ -650,7 +653,11 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	var sz int
+	defer func() {
+		sz = len(buf)
+		done(int64(sz), &err)
+	}()
 
 	return xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result []byte, err error) {
 		return p.storage.ReadAll(ctx, volume, path)
@@ -662,7 +669,9 @@ func (p *xlStorageDiskIDCheck) ReadXL(ctx context.Context, volume string, path s
 	if err != nil {
 		return RawFileInfo{}, err
 	}
-	defer done(&err)
+	defer func() {
+		done(int64(len(rf.Buf)), &err)
+	}()
 
 	return xioutil.WithDeadline[RawFileInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result RawFileInfo, err error) {
 		return p.storage.ReadXL(ctx, volume, path, readData)
@@ -674,7 +683,7 @@ func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path st
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.StatInfoFile(ctx, volume, path, glob)
 }
@@ -689,7 +698,7 @@ func (p *xlStorageDiskIDCheck) ReadMultiple(ctx context.Context, req ReadMultipl
 		xioutil.SafeClose(resp)
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.ReadMultiple(ctx, req, resp)
 }
@@ -701,19 +710,20 @@ func (p *xlStorageDiskIDCheck) CleanAbandonedData(ctx context.Context, volume st
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.CleanAbandonedData(ctx, volume, path) })
 }
 
-func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string, err string, custom map[string]string) madmin.TraceInfo {
+func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string, size int64, err string, custom map[string]string) madmin.TraceInfo {
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceStorage,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  "storage." + s.String(),
 		Duration:  duration,
+		Bytes:     size,
 		Path:      path,
 		Error:     err,
 		Custom:    custom,
@@ -733,10 +743,10 @@ func scannerTrace(s scannerMetric, startTime time.Time, duration time.Duration,
 }
 
 // Update storage metrics
-func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(err *error) {
+func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(sz int64, err *error) {
 	startTime := time.Now()
 	trace := globalTrace.NumSubscribers(madmin.TraceStorage) > 0
-	return func(errp *error) {
+	return func(sz int64, errp *error) {
 		duration := time.Since(startTime)
 
 		var err error
@@ -767,7 +777,7 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 			}
 			custom["total-errs-timeout"] = strconv.FormatUint(p.totalErrsTimeout.Load(), 10)
 			custom["total-errs-availability"] = strconv.FormatUint(p.totalErrsAvailability.Load(), 10)
-			globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " "), errStr, custom))
+			globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " "), sz, errStr, custom))
 		}
 	}
 }
@@ -824,7 +834,7 @@ func (h *healthDiskCtxValue) logSuccess() {
 
 // noopDoneFunc is a no-op done func.
 // Can be reused.
-var noopDoneFunc = func(_ *error) {}
+var noopDoneFunc = func(_ int64, _ *error) {}
 
 // TrackDiskHealth for this request.
 // When a non-nil error is returned 'done' MUST be called
@@ -833,7 +843,7 @@ var noopDoneFunc = func(_ *error) {}
 // is either nil or io.EOF the disk is considered good.
 // So if unsure if the disk status is ok, return nil as a parameter to done.
 // Shadowing will work as long as return error is named: https://go.dev/play/p/sauq86SsTN2
-func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(*error), err error) {
+func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(int64, *error), err error) {
 	done = noopDoneFunc
 	if contextCanceled(ctx) {
 		return ctx, done, ctx.Err()
@@ -866,7 +876,7 @@ func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMet
 	ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess})
 	si := p.updateStorageMetrics(s, paths...)
 	var once sync.Once
-	return ctx, func(errp *error) {
+	return ctx, func(sz int64, errp *error) {
 		p.health.waiting.Add(-1)
 		once.Do(func() {
 			if errp != nil {
@@ -875,7 +885,7 @@ func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMet
 					p.health.logSuccess()
 				}
 			}
-			si(errp)
+			si(sz, errp)
 		})
 	}, nil
 }

cmd/xl-storage-format-utils.go

@@ -23,22 +23,40 @@ import (
 	"github.com/zeebo/xxh3"
 )
 
-func getFileInfoVersions(xlMetaBuf []byte, volume, path string, allParts bool) (FileInfoVersions, error) {
+// getFileInfoVersions partitions this object's versions such that,
+//   - fivs.Versions has all the non-free versions
+//   - fivs.FreeVersions has all the free versions
+//
+// if inclFreeVersions is true all the versions are in fivs.Versions, free and non-free versions alike.
+//
+// Note: Only the scanner requires fivs.Versions to have exclusively non-free versions. This is used while enforcing NewerNoncurrentVersions lifecycle element.
+func getFileInfoVersions(xlMetaBuf []byte, volume, path string, allParts, inclFreeVersions bool) (FileInfoVersions, error) {
 	fivs, err := getAllFileInfoVersions(xlMetaBuf, volume, path, allParts)
 	if err != nil {
 		return fivs, err
 	}
+
+	// If inclFreeVersions is false, partition the versions in fivs.Versions
+	// such that finally fivs.Versions has
+	// all the non-free versions and fivs.FreeVersions has all the free
+	// versions.
 	n := 0
 	for _, fi := range fivs.Versions {
-		// Filter our tier object delete marker
-		if !fi.TierFreeVersion() {
-			fivs.Versions[n] = fi
-			n++
+		// filter our tier object delete marker
+		if fi.TierFreeVersion() {
+			if !inclFreeVersions {
+				fivs.FreeVersions = append(fivs.FreeVersions, fi)
+			}
 		} else {
-			fivs.FreeVersions = append(fivs.FreeVersions, fi)
+			if !inclFreeVersions {
+				fivs.Versions[n] = fi
+			}
+			n++
 		}
 	}
-	fivs.Versions = fivs.Versions[:n]
+	if !inclFreeVersions {
+		fivs.Versions = fivs.Versions[:n]
+	}
 	// Update numversions
 	for i := range fivs.Versions {
 		fivs.Versions[i].NumVersions = n

cmd/xl-storage-format-utils_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"slices"
 	"sort"
 	"testing"
 	"time"
@@ -145,7 +146,7 @@ func TestGetFileInfoVersions(t *testing.T) {
 	}
 	xl := xlMetaV2{}
 	var versions []FileInfo
-	var freeVersionIDs []string
+	var allVersionIDs, freeVersionIDs []string
 	for i := 0; i < 5; i++ {
 		fi := basefi
 		fi.VersionID = mustGetUUID()
@@ -167,18 +168,31 @@ func TestGetFileInfoVersions(t *testing.T) {
 			// delete this version leading to a free version
 			xl.DeleteVersion(fi)
 			freeVersionIDs = append(freeVersionIDs, fi.TierFreeVersionID())
+			allVersionIDs = append(allVersionIDs, fi.TierFreeVersionID())
 		} else {
 			versions = append(versions, fi)
+			allVersionIDs = append(allVersionIDs, fi.VersionID)
 		}
 	}
 	buf, err := xl.AppendTo(nil)
 	if err != nil {
 		t.Fatalf("Failed to serialize xlmeta %v", err)
 	}
-	fivs, err := getFileInfoVersions(buf, basefi.Volume, basefi.Name, true)
+	fivs, err := getFileInfoVersions(buf, basefi.Volume, basefi.Name, true, false)
 	if err != nil {
 		t.Fatalf("getFileInfoVersions failed: %v", err)
 	}
+	chkNumVersions := func(fis []FileInfo) bool {
+		for i := 0; i < len(fis)-1; i++ {
+			if fis[i].NumVersions != fis[i+1].NumVersions {
+				return false
+			}
+		}
+		return true
+	}
+	if !chkNumVersions(fivs.Versions) {
+		t.Fatalf("Expected all versions to have the same NumVersions")
+	}
 
 	sort.Slice(versions, func(i, j int) bool {
 		if versions[i].IsLatest {
@@ -194,6 +208,9 @@ func TestGetFileInfoVersions(t *testing.T) {
 		if fi.VersionID != versions[i].VersionID {
 			t.Fatalf("getFileInfoVersions: versions don't match at %d, version id expected %s but got %s", i, fi.VersionID, versions[i].VersionID)
 		}
+		if fi.NumVersions != len(fivs.Versions) {
+			t.Fatalf("getFileInfoVersions: version with %s version id expected to have %d as NumVersions but got %d", fi.VersionID, len(fivs.Versions), fi.NumVersions)
+		}
 	}
 
 	for i, free := range fivs.FreeVersions {
@@ -201,4 +218,20 @@ func TestGetFileInfoVersions(t *testing.T) {
 			t.Fatalf("getFileInfoVersions: free versions don't match at %d, version id expected %s but got %s", i, free.VersionID, freeVersionIDs[i])
 		}
 	}
+
+	// versions are stored in xl-meta sorted in descending order of their ModTime
+	slices.Reverse(allVersionIDs)
+
+	fivs, err = getFileInfoVersions(buf, basefi.Volume, basefi.Name, true, true)
+	if err != nil {
+		t.Fatalf("getFileInfoVersions failed: %v", err)
+	}
+	if !chkNumVersions(fivs.Versions) {
+		t.Fatalf("Expected all versions to have the same NumVersions")
+	}
+	for i, fi := range fivs.Versions {
+		if fi.VersionID != allVersionIDs[i] {
+			t.Fatalf("getFileInfoVersions: all versions don't match at %d expected %s but got %s", i, allVersionIDs[i], fi.VersionID)
+		}
+	}
 }

cmd/xl-storage-format-v2.go

@@ -250,7 +250,7 @@ type xlMetaV2VersionHeader struct {
 	Signature [4]byte
 	Type      VersionType
 	Flags     xlFlags
-	EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
+	EcN, EcM  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 }
 
 func (x xlMetaV2VersionHeader) String() string {
@@ -368,8 +368,8 @@ func (j *xlMetaV2Version) header() xlMetaV2VersionHeader {
 		Signature: j.getSignature(),
 		Type:      j.Type,
 		Flags:     flags,
-		EcN:       ecM,
-		EcM:       ecN,
+		EcN:       ecN,
+		EcM:       ecM,
 	}
 }
 

cmd/xl-storage-format-v2_gen.go

@@ -2235,16 +2235,16 @@ func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		z.Flags = xlFlags(zb0003)
 	}
-	z.EcM, err = dc.ReadUint8()
-	if err != nil {
-		err = msgp.WrapError(err, "EcM")
-		return
-	}
 	z.EcN, err = dc.ReadUint8()
 	if err != nil {
 		err = msgp.WrapError(err, "EcN")
 		return
 	}
+	z.EcM, err = dc.ReadUint8()
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
 	return
 }
 
@@ -2280,16 +2280,16 @@ func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Flags")
 		return
 	}
-	err = en.WriteUint8(z.EcM)
-	if err != nil {
-		err = msgp.WrapError(err, "EcM")
-		return
-	}
 	err = en.WriteUint8(z.EcN)
 	if err != nil {
 		err = msgp.WrapError(err, "EcN")
 		return
 	}
+	err = en.WriteUint8(z.EcM)
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
 	return
 }
 
@@ -2303,8 +2303,8 @@ func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendBytes(o, (z.Signature)[:])
 	o = msgp.AppendUint8(o, uint8(z.Type))
 	o = msgp.AppendUint8(o, uint8(z.Flags))
-	o = msgp.AppendUint8(o, z.EcM)
 	o = msgp.AppendUint8(o, z.EcN)
+	o = msgp.AppendUint8(o, z.EcM)
 	return
 }
 
@@ -2353,16 +2353,16 @@ func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		z.Flags = xlFlags(zb0003)
 	}
-	z.EcM, bts, err = msgp.ReadUint8Bytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err, "EcM")
-		return
-	}
 	z.EcN, bts, err = msgp.ReadUint8Bytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err, "EcN")
 		return
 	}
+	z.EcM, bts, err = msgp.ReadUint8Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
 	o = bts
 	return
 }

cmd/xl-storage.go

@@ -586,7 +586,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		// Remove filename which is the meta file.
 		item.transformMetaDir()
 
-		fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath(), false)
+		fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath(), false, false)
 		metaDataPoolPut(buf)
 		if err != nil {
 			res["err"] = err.Error()
@@ -2312,18 +2312,25 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string,
 }
 
 // CheckParts check if path has necessary parts available.
-func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
+func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) {
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	for _, part := range fi.Parts {
+	err = checkPathLength(pathJoin(volumeDir, path))
+	if err != nil {
+		return nil, err
+	}
+
+	resp := CheckPartsResp{
+		// By default, all results have an unknown status
+		Results: make([]int, len(fi.Parts)),
+	}
+
+	for i, part := range fi.Parts {
 		partPath := pathJoin(path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
 		filePath := pathJoin(volumeDir, partPath)
-		if err = checkPathLength(filePath); err != nil {
-			return err
-		}
 		st, err := Lstat(filePath)
 		if err != nil {
 			if osIsNotExist(err) {
@@ -2331,24 +2338,30 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string,
 					// Stat a volume entry.
 					if verr := Access(volumeDir); verr != nil {
 						if osIsNotExist(verr) {
-							return errVolumeNotFound
+							resp.Results[i] = checkPartVolumeNotFound
 						}
-						return verr
+						continue
 					}
 				}
 			}
-			return osErrToFileErr(err)
+			if osErrToFileErr(err) == errFileNotFound {
+				resp.Results[i] = checkPartFileNotFound
+			}
+			continue
 		}
 		if st.Mode().IsDir() {
-			return errFileNotFound
+			resp.Results[i] = checkPartFileNotFound
+			continue
 		}
 		// Check if shard is truncated.
 		if st.Size() < fi.Erasure.ShardFileSize(part.Size) {
-			return errFileCorrupt
+			resp.Results[i] = checkPartFileCorrupt
+			continue
 		}
+		resp.Results[i] = checkPartSuccess
 	}
 
-	return nil
+	return &resp, nil
 }
 
 // deleteFile deletes a file or a directory if its empty unless recursive
@@ -2922,42 +2935,43 @@ func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize
 	return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize)
 }
 
-func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) {
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if !skipAccessChecks(volume) {
 		// Stat a volume entry.
 		if err = Access(volumeDir); err != nil {
-			return convertAccessError(err, errVolumeAccessDenied)
+			return nil, convertAccessError(err, errVolumeAccessDenied)
 		}
 	}
 
+	resp := CheckPartsResp{
+		// By default, the result is unknown per part
+		Results: make([]int, len(fi.Parts)),
+	}
+
 	erasure := fi.Erasure
-	for _, part := range fi.Parts {
+	for i, part := range fi.Parts {
 		checksumInfo := erasure.GetChecksumInfo(part.Number)
 		partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
-		if err := s.bitrotVerify(ctx, partPath,
+		err := s.bitrotVerify(ctx, partPath,
 			erasure.ShardFileSize(part.Size),
 			checksumInfo.Algorithm,
-			checksumInfo.Hash, erasure.ShardSize()); err != nil {
-			if !IsErr(err, []error{
-				errFileNotFound,
-				errVolumeNotFound,
-				errFileCorrupt,
-				errFileAccessDenied,
-				errFileVersionNotFound,
-			}...) {
-				logger.GetReqInfo(ctx).AppendTags("disk", s.String())
-				storageLogOnceIf(ctx, err, partPath)
-			}
-			return err
+			checksumInfo.Hash, erasure.ShardSize())
+
+		resp.Results[i] = convPartErrToInt(err)
+
+		// Only log unknown errors
+		if resp.Results[i] == checkPartUnknown && err != errFileAccessDenied {
+			logger.GetReqInfo(ctx).AppendTags("disk", s.String())
+			storageLogOnceIf(ctx, err, partPath)
 		}
 	}
 
-	return nil
+	return &resp, nil
 }
 
 // ReadMultiple will read multiple files and send each back as response.

docker-buildx.sh

@@ -2,6 +2,14 @@
 
 sudo sysctl net.ipv6.conf.all.disable_ipv6=0
 
+remote=$(git remote get-url upstream)
+if test "$remote" != "git@github.com:minio/minio.git"; then
+	echo "Script requires that the 'upstream' remote is set to git@github.com:minio/minio.git"
+	exit 1
+fi
+
+git remote update upstream && git checkout master && git rebase upstream/master
+
 release=$(git describe --abbrev=0 --tags)
 
 docker buildx build --push --no-cache \

docs/bucket/replication/test_del_marker_proxying.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+	cleanup
+
+	for site in sitea siteb; do
+		echo "$site server logs ========="
+		cat "/tmp/${site}_1.log"
+		echo "==========================="
+		cat "/tmp/${site}_2.log"
+	done
+
+	exit 1
+}
+
+cleanup() {
+	echo -n "Cleaning up instances of MinIO ..."
+	pkill -9 minio || sudo pkill -9 minio
+	rm -rf /tmp/sitea
+	rm -rf /tmp/siteb
+	echo "done"
+}
+
+cleanup
+
+export MINIO_CI_CD=1
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+
+# Start MinIO instances
+echo -n "Starting MinIO instances ..."
+minio server --address ":9001" --console-address ":10000" /tmp/sitea/{1...4}/disk{1...4} /tmp/sitea/{5...8}/disk{1...4} >/tmp/sitea_1.log 2>&1 &
+minio server --address ":9002" --console-address ":11000" /tmp/siteb/{1...4}/disk{1...4} /tmp/siteb/{5...8}/disk{1...4} >/tmp/siteb_1.log 2>&1 &
+echo "done"
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001
+export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9002
+
+./mc ready sitea
+./mc ready siteb
+
+./mc mb sitea/bucket
+./mc version enable sitea/bucket
+./mc mb siteb/bucket
+./mc version enable siteb/bucket
+
+# Set bucket replication
+./mc replicate add sitea/bucket --remote-bucket siteb/bucket
+
+# Run the test to make sure proxying of DEL marker doesn't happen
+loop_count=0
+while true; do
+	if [ $loop_count -eq 100 ]; then
+		break
+	fi
+	echo "Hello World" | ./mc pipe sitea/bucket/obj$loop_count
+	./mc rm sitea/bucket/obj$loop_count
+	RESULT=$({ ./mc stat sitea/bucket/obj$loop_count; } 2>&1)
+	if [[ ${RESULT} != *"Object does not exist"* ]]; then
+		echo "BUG: stat should fail. succeeded."
+		exit_1
+	fi
+	loop_count=$((loop_count + 1))
+done
+
+cleanup

docs/debugging/inspect/main.go

@@ -59,7 +59,7 @@ func main() {
 		}
 
 		// Prompt for decryption key if no --key or --private-key are provided
-		if len(privateKey) == 0 {
+		if len(privateKey) == 0 && !*stdin {
 			reader := bufio.NewReader(os.Stdin)
 			fmt.Print("Enter Decryption Key: ")
 

docs/debugging/xl-meta/main.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
@@ -710,7 +711,7 @@ type xlMetaV2VersionHeaderV2 struct {
 	Signature [4]byte
 	Type      uint8
 	Flags     uint8
-	EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
+	EcN, EcM  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
@@ -768,19 +769,19 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt
 			var zb0004 uint8
 			zb0004, bts, err = msgp.ReadUint8Bytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "EcM")
+				err = msgp.WrapError(err, "EcN")
 				return
 			}
-			z.EcM = zb0004
+			z.EcN = zb0004
 		}
 		{
 			var zb0005 uint8
 			zb0005, bts, err = msgp.ReadUint8Bytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "EcN")
+				err = msgp.WrapError(err, "EcM")
 				return
 			}
-			z.EcN = zb0005
+			z.EcM = zb0005
 		}
 	}
 	o = bts
@@ -814,6 +815,7 @@ type mappedData struct {
 	blockOffset                int // Offset in bytes to start of block.
 	blocks                     int // 0 = one block.
 	objSize, partSize          int
+	wantMD5                    string
 }
 
 func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
@@ -835,6 +837,9 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 				EcBSize   int
 				PartNums  []int
 				PartSizes []int
+				MetaUsr   struct {
+					Etag string `json:"etag"`
+				}
 			}
 		}
 		var ei erasureInfo
@@ -845,11 +850,14 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 			}
 			m.data = ei.V2Obj.EcM
 			m.parity = ei.V2Obj.EcN
+			if len(ei.V2Obj.PartNums) == 1 && !strings.ContainsRune(ei.V2Obj.MetaUsr.Etag, '-') {
+				m.wantMD5 = ei.V2Obj.MetaUsr.Etag
+			}
 			if m.shards == 0 {
 				m.shards = m.data + m.parity
 			}
 			idx = ei.V2Obj.EcIndex - 1
-			fmt.Println("Read shard", ei.V2Obj.EcIndex, "Data shards", m.data, "Parity", m.parity, fmt.Sprintf("(%s)", file))
+			fmt.Println("Read shard", ei.V2Obj.EcIndex, fmt.Sprintf("(%s)", file))
 			if ei.V2Obj.Size != m.objSize {
 				return nil, fmt.Errorf("size mismatch. Meta size: %d, Prev: %d", ei.V2Obj.Size, m.objSize)
 			}
@@ -893,7 +901,6 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 		if err != nil {
 			return nil, err
 		}
-		fmt.Println("Block data size:", m.size, "Shard size", ssz, "Got Shard:", len(b), "Bitrot ok")
 
 		if m.mapped == nil {
 			m.mapped = make([]byte, m.size)
@@ -912,6 +919,7 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 		if start >= len(m.mapped) {
 			continue
 		}
+		fmt.Println("Block data size:", m.size, "Shard size", ssz, "Got Shard:", len(b), "Bitrot ok", "Start", start, "End", start+len(b))
 		copy(m.mapped[start:], b)
 		for j := range b {
 			if j+start >= len(m.filled) {
@@ -1038,14 +1046,23 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 		if len(files[partIdx]) == 0 {
 			continue
 		}
+		var wantMD5 string
 		exportedSizes := make(map[int]bool)
+		// block -> data
+		combineSharedBlocks := make(map[int][]byte)
+		combineFilledBlocks := make(map[int][]byte)
 	nextFile:
 		for key, file := range files[partIdx] {
 			fmt.Println("Reading base version", file[0], "part", part)
 			var combined []byte
 			var missingAll int
 			var lastValidAll int
+
+			attempt := 0
 			for block := 0; ; block++ {
+				combineFilled := combineFilledBlocks[block]
+				combineShared := combineSharedBlocks[block]
+			nextAttempt:
 				fmt.Printf("Block %d, Base version %q. Part %d. Files %d\n", block+1, key, part, len(file))
 				m, err := readAndMap(file, part, block)
 				if err != nil {
@@ -1055,11 +1072,30 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 					fmt.Println("Skipping version", key, "as it has already been exported.")
 					continue nextFile
 				}
+				addedFiles := 0
 			compareFile:
 				for otherKey, other := range files[partIdx] {
+					addedFiles++
+					if attempt > 0 && len(m.filled) == len(combineFilled) {
+						fmt.Println("Merging previous global data")
+						filled := 0
+						missing := 0
+						for i, v := range combineFilled {
+							if v == 1 {
+								m.filled[i] = 1
+								m.mapped[i] = combineShared[i]
+								filled++
+							} else {
+								missing++
+							}
+						}
+						fmt.Println("Missing", missing, "bytes. Filled", filled, "bytes.")
+						break
+					}
 					if key == otherKey {
 						continue
 					}
+
 					otherPart := getPartNum(other[0])
 					if part != otherPart {
 						fmt.Println("part ", part, " != other part", otherPart, other[0])
@@ -1076,16 +1112,6 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 					if m.objSize != otherM.objSize {
 						continue
 					}
-					var ok int
-					for i, filled := range otherM.filled[:m.size] {
-						if filled == 1 && m.filled[i] == 1 {
-							if m.mapped[i] != otherM.mapped[i] {
-								fmt.Println("Data mismatch at byte", i, "-  Disregarding version", otherKey)
-								continue compareFile
-							}
-							ok++
-						}
-					}
 
 					// If data+parity matches, combine.
 					if m.parity == otherM.parity && m.data == otherM.data {
@@ -1102,6 +1128,17 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						}
 					}
 
+					var ok int
+					for i, filled := range otherM.filled[:m.size] {
+						if filled == 1 && m.filled[i] == 1 {
+							if m.mapped[i] != otherM.mapped[i] {
+								fmt.Println("Data mismatch at byte", i, "-  Disregarding version", otherKey)
+								continue compareFile
+							}
+							ok++
+						}
+					}
+
 					fmt.Printf("Data overlaps (%d bytes). Combining with %q.\n", ok, otherKey)
 					for i := range otherM.filled {
 						if otherM.filled[i] == 1 {
@@ -1110,6 +1147,7 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						}
 					}
 				}
+
 				lastValid := 0
 				missing := 0
 				for i := range m.filled {
@@ -1130,7 +1168,7 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						if err != nil {
 							return err
 						}
-						split, err := rs.Split(m.mapped)
+						splitData, err := rs.Split(m.mapped)
 						if err != nil {
 							return err
 						}
@@ -1138,35 +1176,172 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						if err != nil {
 							return err
 						}
-						ok := len(splitFilled)
-						for i, sh := range splitFilled {
-							for _, v := range sh {
+						// Fill padding...
+						padding := len(splitFilled[0])*k - len(m.filled)
+						for i := 0; i < padding; i++ {
+							arr := splitFilled[k-1]
+							arr[len(arr)-i-1] = 1
+						}
+
+						hasParity := 0
+						parityOK := make([]bool, m.shards)
+						for idx, sh := range v {
+							splitData[idx] = sh
+							if idx >= k && len(sh) > 0 {
+								parityOK[idx] = true
+								hasParity++
+								for i := range splitFilled[idx] {
+									splitFilled[idx][i] = 1
+								}
+							}
+						}
+
+						splitDataShards := make([]byte, len(splitFilled[0]))
+						for _, sh := range splitFilled {
+							for i, v := range sh {
+								splitDataShards[i] += v
+							}
+						}
+						var hist [256]int
+						for _, v := range splitDataShards {
+							hist[v]++
+						}
+
+						for _, v := range hist[m.data-hasParity : m.shards] {
+							if attempt > 0 {
+								break
+							}
+							if v == 0 {
+								continue
+							}
+							for i, v := range hist[:m.shards] {
+								if v > 0 {
+									if i < m.data {
+										fmt.Println("- Shards:", i, "of", m.data, "Bytes:", v, "Missing: ", v*(m.data-i+hasParity))
+									} else {
+										fmt.Println("+ Shards:", i, "of", m.data, "Bytes:", v, "Recovering: ", v*(m.data-i+hasParity))
+									}
+								}
+							}
+							fmt.Println("Attempting to reconstruct with partial shards")
+							offset := 0
+							startOffset := 0
+							shardConfig := make([]byte, k)
+							reconstructAbleConfig := false
+							shards := make([][]byte, m.shards)
+							for i := range shards {
+								shards[i] = make([]byte, 0, len(splitData[0]))
+							}
+							for offset < len(splitDataShards) {
+								newConfig := false
+								for shardIdx, shard := range splitFilled[:k] {
+									if shardConfig[shardIdx] != shard[offset] {
+										newConfig = true
+										break
+									}
+								}
+								if newConfig {
+									if offset > startOffset && reconstructAbleConfig {
+										reconPartial(shards, k, parityOK, splitData, startOffset, offset, rs, shardConfig, splitFilled)
+									}
+									// Update to new config and add current
+									valid := 0
+									for shardIdx, shard := range splitFilled[:k] {
+										shardConfig[shardIdx] = shard[offset]
+										valid += int(shard[offset])
+										if shard[offset] == 0 {
+											shards[shardIdx] = shards[shardIdx][:0]
+										} else {
+											shards[shardIdx] = append(shards[shardIdx][:0], splitData[shardIdx][offset])
+										}
+									}
+									reconstructAbleConfig = valid >= m.data-hasParity && valid < m.data
+									startOffset = offset
+									offset++
+									continue
+								}
+								for shardIdx, ok := range shardConfig {
+									if ok != 0 {
+										shards[shardIdx] = append(shards[shardIdx], splitData[shardIdx][offset])
+									}
+								}
+								offset++
+							}
+							if offset > startOffset && reconstructAbleConfig {
+								reconPartial(shards, k, parityOK, splitData, startOffset, offset, rs, shardConfig, splitFilled)
+							}
+
+							var buf bytes.Buffer
+							if err := rs.Join(&buf, splitFilled, m.size); err == nil {
+								m.filled = buf.Bytes()
+							}
+							buf = bytes.Buffer{}
+							if err := rs.Join(&buf, splitData, m.size); err == nil {
+								m.mapped = buf.Bytes()
+							}
+							for i, v := range m.filled {
 								if v == 0 {
-									split[i] = nil
-									ok--
+									m.mapped[i] = 0
+								}
+							}
+							break
+						}
+						ok := k
+						for i, sh := range splitFilled {
+							for j, v := range sh {
+								if v == 0 {
+									splitData[i] = nil
+									if i < k {
+										fmt.Println("Shard", i, "is missing data from offset", i*len(sh)+j)
+										ok--
+									}
 									break
 								}
 							}
 						}
-						hasParity := 0
-						for idx, sh := range v {
-							split[idx] = sh
-							if idx >= k && len(sh) > 0 {
-								hasParity++
+
+						missing = 0
+						lastValid = 0
+						for i := range m.filled {
+							if m.filled[i] == 1 {
+								lastValid = i
+							} else {
+								missing++
 							}
 						}
-						fmt.Printf("Have %d complete remapped data shards and %d complete parity shards. ", ok, hasParity)
+						fmt.Printf("Have %d complete remapped data shards and %d complete parity shards (%d bytes missing). ", ok, hasParity, missing)
 
-						if err := rs.ReconstructData(split); err == nil {
-							fmt.Println("Could reconstruct completely")
-							for i, data := range split[:k] {
+						if err := rs.ReconstructData(splitData); err == nil {
+							fmt.Println("Could reconstruct completely.")
+							for i, data := range splitData[:k] {
 								start := i * len(data)
 								copy(m.mapped[start:], data)
 							}
 							lastValid = m.size - 1
 							missing = 0
+							attempt = 2
+							wantMD5 = m.wantMD5
 						} else {
-							fmt.Println("Could NOT reconstruct:", err)
+							fmt.Println("Could NOT reconstruct:", err, " - Need", m.data, "shards.")
+							if attempt == 0 {
+								if len(combineShared) == 0 {
+									combineShared = make([]byte, len(m.mapped))
+									combineFilled = make([]byte, len(m.filled))
+								}
+								for i := range m.filled {
+									if m.filled[i] == 1 && combineFilled[i] == 0 {
+										combineShared[i] = m.mapped[i]
+										combineFilled[i] = 1
+									}
+								}
+								combineFilledBlocks[block] = combineFilled
+								combineSharedBlocks[block] = combineShared
+								fmt.Println("Retrying with merged data")
+								if addedFiles >= len(files[partIdx]) {
+									attempt++
+									goto nextAttempt
+								}
+							}
 						}
 					}
 				}
@@ -1182,7 +1357,7 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 					if len(combined) != m.partSize {
 						fmt.Println("Combined size mismatch. Expected", m.partSize, "got", len(combined))
 					}
-					fmt.Println("Reached block", block+1, "of", m.blocks+1, "for", key, ". Done.")
+					fmt.Println("Reached block", block+1, "of", m.blocks+1, "for", key, "Done.")
 					break
 				}
 			}
@@ -1195,16 +1370,31 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 			}
 			if missingAll > 0 {
 				out += ".incomplete"
-				fmt.Println(missingAll, "bytes missing. Truncating", len(combined)-lastValidAll-1, "from end.")
+				fmt.Println(missingAll, "bytes missing.")
 			} else {
-				out += ".complete"
-				fmt.Println("No bytes missing.")
+				if wantMD5 != "" {
+					sum := md5.Sum(combined)
+					gotMD5 := hex.EncodeToString(sum[:])
+					if gotMD5 != wantMD5 {
+						fmt.Println("MD5 mismatch. Expected", wantMD5, "got", gotMD5)
+						out += ".mismatch"
+					} else {
+						fmt.Println("MD5 verified.")
+						out = fmt.Sprintf("verified/%s", baseName)
+					}
+				} else {
+					out = fmt.Sprintf("complete/%s.%05d", baseName, part)
+					fmt.Println("No bytes missing.")
+				}
 			}
 			if missingAll == 0 {
 				exportedSizes[len(combined)] = true
 			}
-			combined = combined[:lastValidAll+1]
-			err := os.WriteFile(out, combined, os.ModePerm)
+			err := os.MkdirAll(filepath.Dir(out), os.ModePerm)
+			if err != nil {
+				return err
+			}
+			err = os.WriteFile(out, combined, os.ModePerm)
 			if err != nil {
 				return err
 			}
@@ -1214,6 +1404,46 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 	return nil
 }
 
+func reconPartial(shards [][]byte, k int, parityOK []bool, splitData [][]byte, startOffset int, offset int, rs reedsolomon.Encoder, shardConfig []byte, splitFilled [][]byte) {
+	// Add parity
+	for i := range shards[k:] {
+		shards[i+k] = nil
+		if parityOK[i+k] {
+			shards[i+k] = splitData[i+k][startOffset:offset]
+		}
+	}
+	// Reconstruct with current config.
+	if err := rs.ReconstructData(shards); err != nil {
+		panic(fmt.Sprintln("Internal error, could NOT partially reconstruct:", err))
+	}
+	// Copy reconstructed data back.
+	verified := 0
+	reconstructed := 0
+	for shardsIdx, ok := range shardConfig {
+		if ok == 0 {
+			copy(splitData[shardsIdx][startOffset:], shards[shardsIdx])
+			for i := range shards[shardsIdx] {
+				if splitFilled[shardsIdx][startOffset+i] == 1 {
+					fmt.Println("Internal error: Found filled data at", startOffset+i)
+				}
+				splitFilled[shardsIdx][startOffset+i] = 1
+			}
+			reconstructed += len(shards[shardsIdx])
+		} else {
+			for i := range shards[shardsIdx] {
+				if splitFilled[shardsIdx][startOffset+i] == 0 {
+					fmt.Println("Internal error: Expected filled data at", startOffset+i)
+				}
+				if splitData[shardsIdx][startOffset+i] != shards[shardsIdx][i] {
+					fmt.Println("Internal error: Mismatch at", startOffset+i)
+				}
+				verified++
+			}
+		}
+	}
+	fmt.Println("Reconstructed", reconstructed, "bytes and verified", verified, "bytes of partial shard with config", shardConfig)
+}
+
 // bitrot returns a shard beginning at startOffset after doing bitrot checks.
 func bitrot(val []byte, startOffset, shardSize int) ([]byte, error) {
 	var res []byte

docs/metrics/prometheus/grafana/node/minio-node.json

@@ -394,32 +394,7 @@
           },
           "unit": "none"
         },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "[play.min.io:9000:/disk4/data]"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
       },
       "gridPos": {
         "h": 5,

docs/metrics/prometheus/grafana/replication/minio-replication-node.json

@@ -162,7 +162,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_average_active_workers{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_average_active_workers{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -290,7 +290,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server, endpoint) (minio_cluster_replication_average_link_latency_ms{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server, endpoint) (minio_node_replication_average_link_latency_ms{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server,endpoint}}",
@@ -418,7 +418,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_average_queued_bytes{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_average_queued_bytes{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -546,7 +546,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_average_queued_count{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_average_queued_count{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -674,7 +674,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_average_transfer_rate{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_average_transfer_rate{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -802,7 +802,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_current_active_workers{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_current_active_workers{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -930,7 +930,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server,endpoint) (minio_cluster_replication_current_link_latency_ms{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server,endpoint) (minio_node_replication_current_link_latency_ms{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -1058,7 +1058,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_current_transfer_rate{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_current_transfer_rate{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -1186,7 +1186,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_last_minute_queued_bytes{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_last_minute_queued_bytes{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server}}",
@@ -1314,7 +1314,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server) (minio_cluster_replication_last_minute_queued_count{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server) (minio_node_replication_last_minute_queued_count{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{bucket}}",
@@ -1442,7 +1442,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server,endpoint) (minio_cluster_replication_link_downtime_duration_seconds{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server,endpoint) (minio_node_replication_link_downtime_duration_seconds{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server,endpoint}}",
@@ -1540,7 +1540,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server,endpoint) (minio_cluster_replication_link_offline_duration_seconds{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server,endpoint) (minio_node_replication_link_offline_duration_seconds{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "legendFormat": "{{server,endpoint}}",
           "refId": "A"
@@ -1637,7 +1637,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "minio_cluster_replication_max_active_workers{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_replication_max_active_workers{job=\"$scrape_jobs\"}",
           "interval": "1m",
           "legendFormat": "{{server}}",
           "refId": "A"
@@ -1734,7 +1734,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server,endpoibt) (minio_cluster_replication_max_link_latency_ms{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server,endpoibt) (minio_node_replication_max_link_latency_ms{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "legendFormat": "{{server,endpoint}}",
           "refId": "A"
@@ -1831,7 +1831,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "minio_cluster_replication_max_queued_bytes{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_replication_max_queued_bytes{job=\"$scrape_jobs\"}",
           "interval": "1m",
           "legendFormat": "{{server}}",
           "refId": "A"
@@ -1928,7 +1928,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "minio_cluster_replication_max_queued_count{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_replication_max_queued_count{job=\"$scrape_jobs\"}",
           "interval": "1m",
           "legendFormat": "{{server}}",
           "refId": "A"
@@ -2025,7 +2025,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "minio_cluster_replication_max_transfer_rate{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_replication_max_transfer_rate{job=\"$scrape_jobs\"}",
           "interval": "1m",
           "legendFormat": "{{server}}",
           "refId": "A"
@@ -2122,7 +2122,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "minio_cluster_replication_recent_backlog_count{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_replication_recent_backlog_count{job=\"$scrape_jobs\"}",
           "interval": "1m",
           "legendFormat": "{{server}}",
           "refId": "A"
@@ -2219,7 +2219,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (server,endpoint) (minio_cluster_replication_link_online{job=\"$scrape_jobs\"})",
+          "expr": "sum by (server,endpoint) (minio_node_replication_link_online{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "legendFormat": "{{endpoint}}",
           "refId": "A"
@@ -2316,7 +2316,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "minio_cluster_replication_link_offline_duration_seconds{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_replication_link_offline_duration_seconds{job=\"$scrape_jobs\"}",
           "interval": "1m",
           "legendFormat": "{{endpoint}}",
           "refId": "A"