replication: pass checksum headers to replica (#19834)

Signed-off-by: Bala.FA <bala@minio.io>

.github/workflows/mint.yml

@@ -55,9 +55,10 @@ jobs:
         run: |
           ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
 
-      - name: resiliency
-        run: |
-          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+      # FIXME: renable this back when we have a valid way to add deadlines for PUT()s (internode CreateFile)
+      # - name: resiliency
+      #   run: |
+      #     ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
 
       - name: The job must cleanup
         if: ${{ always() }}

.github/workflows/vulncheck.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.3
+          go-version: 1.22.4
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

Makefile

@@ -106,7 +106,10 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
+test-delete-marker-proxying:
+	@(env bash $(PWD)/docs/bucket/replication/test_del_marker_proxying.sh)
+
+test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error test-delete-marker-proxying ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
 test-site-replication-ldap: install-race ## verify automatic site replication

buildscripts/verify-healing-empty-erasure-set.sh

@@ -38,8 +38,40 @@ function start_minio_3_node() {
 	disown $pid3
 
 	export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
+
 	/tmp/mc ready myminio
 
+	# Wait for all drives to be online and formatted
+	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].state | select(. != "ok")' | wc -l) -gt 0 ]; do sleep 1; done
+	# Wait for all drives to be healed
+	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].healing | select(. != null) | select(. == true)' | wc -l) -gt 0 ]; do sleep 1; done
+
+	# Wait for Status: in MinIO output
+	while true; do
+		rv=$(check_online)
+		if [ "$rv" != "1" ]; then
+			# success
+			break
+		fi
+
+		# Check if we should retry
+		retry=$((retry + 1))
+		if [ $retry -le 20 ]; then
+			sleep 5
+			continue
+		fi
+
+		# Failure
+		for i in $(seq 1 3); do
+			echo "server$i log:"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
+		done
+		pkill -9 minio
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	done
+
 	if ! ps -p $pid1 1>&2 >/dev/null; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/dist-minio-server1.log"
@@ -90,7 +122,7 @@ function check_online() {
 }
 
 function purge() {
-	rm -rf "$1"
+	echo rm -rf "$1"
 }
 
 function __init__() {
@@ -117,18 +149,6 @@ function perform_test() {
 
 	set -x
 	start_minio_3_node $2
-
-	rv=$(check_online)
-	if [ "$rv" == "1" ]; then
-		for i in $(seq 1 3); do
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		pkill -9 minio
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
 }
 
 function main() {

cmd/admin-handlers-idp-ldap.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
@@ -284,6 +285,18 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 		opts.claims[ldapUser] = targetUser // DN
 		opts.claims[ldapActualUser] = lookupResult.ActualDN
 
+		// Check if this user or their groups have a policy applied.
+		ldapPolicies, err := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if len(ldapPolicies) == 0 {
+			err = fmt.Errorf("No policy set for user `%s` or any of their groups: `%s`", opts.claims[ldapActualUser], strings.Join(targetGroups, "`,`"))
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL)
+			return
+		}
+
 		// Add LDAP attributes that were looked up into the claims.
 		for attribKey, attribValue := range lookupResult.Attributes {
 			opts.claims[ldapAttribPrefix+attribKey] = attribValue

cmd/api-errors.go

@@ -967,7 +967,7 @@ var errorCodes = errorCodeMap{
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
 		Description:    "Remote service connection error",
-		HTTPStatusCode: http.StatusNotFound,
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrReplicationBandwidthLimitError: {
 		Code:           "XMinioAdminReplicationBandwidthLimitError",

cmd/api-response.go

@@ -789,7 +789,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 }
 
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
-func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
+func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
 	cs := oi.decryptChecksums(0)
 	c := CompleteMultipartUploadResponse{
 		Location: location,

cmd/background-newdisks-heal-ops.go

@@ -141,14 +141,14 @@ func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	return h
 }
 
-func (h healingTracker) getLastUpdate() time.Time {
+func (h *healingTracker) getLastUpdate() time.Time {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
 	return h.LastUpdate
 }
 
-func (h healingTracker) getBucket() string {
+func (h *healingTracker) getBucket() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
@@ -162,7 +162,7 @@ func (h *healingTracker) setBucket(bucket string) {
 	h.Bucket = bucket
 }
 
-func (h healingTracker) getObject() string {
+func (h *healingTracker) getObject() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 
@@ -453,10 +453,6 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 
 	healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 
-	if len(tracker.QueuedBuckets) > 0 {
-		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
-	}
-
 	if serverDebugLog {
 		tracker.printTo(os.Stdout)
 		fmt.Printf("\n")

cmd/batch-handlers.go

@@ -1476,7 +1476,7 @@ func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string
 
 func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
 	// TODO: support custom storage class for remote replication
-	putOpts, err = putReplicationOpts(ctx, "", objInfo)
+	putOpts, err = putReplicationOpts(ctx, "", objInfo, 0)
 	if err != nil {
 		return putOpts, err
 	}
@@ -1633,7 +1633,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	job.ID = fmt.Sprintf("%s:%d", shortuuid.New(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
+	job.ID = fmt.Sprintf("%s%s%d", shortuuid.New(), getBatchJobIDSeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
 

cmd/batch-job-common-types.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"fmt"
+	"runtime"
 	"strings"
 	"time"
 
@@ -288,3 +289,12 @@ func (s *BatchJobSize) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*s = BatchJobSize(sz)
 	return nil
 }
+
+// getBatchJobIDSeparator - returns the separator to be used in the batch job ID
+// windows requires `_` as the separator `:` will be an invalid one
+func getBatchJobIDSeparator() string {
+	if runtime.GOOS == globalWindowsOSName {
+		return "_"
+	}
+	return ":"
+}

cmd/bucket-handlers.go

@@ -1420,7 +1420,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if formValues.Get(postPolicyBucketTagging) != "" {
 		tags, err := tags.ParseObjectXML(strings.NewReader(formValues.Get(postPolicyBucketTagging)))
 		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 			return
 		}
 		tagsStr := tags.String()

cmd/bucket-lifecycle.go

@@ -72,6 +72,7 @@ func NewLifecycleSys() *LifecycleSys {
 }
 
 func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+	sz, _ := oi.GetActualSize()
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceILM,
 		Time:      startTime,
@@ -79,6 +80,7 @@ func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event
 		FuncName:  event,
 		Duration:  duration,
 		Path:      pathJoin(oi.Bucket, oi.Name),
+		Bytes:     sz,
 		Error:     "",
 		Message:   getSource(4),
 		Custom:    map[string]string{"version-id": oi.VersionID},
@@ -277,6 +279,10 @@ func (es *expiryState) getWorkerCh(h uint64) chan<- expiryOp {
 }
 
 func (es *expiryState) ResizeWorkers(n int) {
+	if n == 0 {
+		n = 100
+	}
+
 	// Lock to avoid multiple resizes to happen at the same time.
 	es.mu.Lock()
 	defer es.mu.Unlock()
@@ -538,6 +544,10 @@ func (t *transitionState) UpdateWorkers(n int) {
 }
 
 func (t *transitionState) updateWorkers(n int) {
+	if n == 0 {
+		n = 100
+	}
+
 	for t.numWorkers < n {
 		go t.worker(t.objAPI)
 		t.numWorkers++

cmd/bucket-listobjects-handlers.go

@@ -231,7 +231,7 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1
 	}
-	i := strings.Index(token, ":")
+	i := strings.Index(token, getBatchJobIDSeparator())
 	if i < 0 {
 		return token, -1
 	}

cmd/bucket-replication.go

@@ -763,12 +763,32 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
-func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
+func getCRCMeta(oi ObjectInfo, partNum int) map[string]string {
+	meta := make(map[string]string)
+	cs := oi.decryptChecksums(partNum)
+	for k, v := range cs {
+		cksum := hash.NewChecksumString(k, v)
+		if cksum == nil {
+			continue
+		}
+		if cksum.Valid() {
+			meta[cksum.Type.Key()] = v
+		}
+	}
+	return meta
+}
+
+func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
 		// In case of SSE-C objects copy the allowed internal headers as well
 		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
 			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+				if strings.EqualFold(k, ReservedMetadataPrefixLower+"crc") {
+					for k, v := range getCRCMeta(objInfo, partNum) {
+						meta[k] = v
+					}
+				}
 				continue
 			}
 			if isStandardHeader(k) {
@@ -782,6 +802,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
 		}
 	}
 
+	if len(objInfo.Checksum) > 0 {
+		for k, v := range getCRCMeta(objInfo, 0) {
+			meta[k] = v
+		}
+	}
+
 	if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) {
 		sc = objInfo.StorageClass
 	}
@@ -1239,7 +1265,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	// use core client to avoid doing multipart on PUT
 	c := &minio.Core{Client: tgt.Client}
 
-	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
+	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
 	if err != nil {
 		replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
 		sendEvent(eventArgs{
@@ -1506,7 +1532,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
-		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
+		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo, 0)
 		if err != nil {
 			replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
 			sendEvent(eventArgs{
@@ -1614,6 +1640,10 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 
 		cHeader := http.Header{}
 		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
+		crc := getCRCMeta(objInfo, partInfo.Number)
+		for k, v := range crc {
+			cHeader.Add(k, v)
+		}
 		popts := minio.PutObjectPartOptions{
 			SSE:          opts.ServerSideEncryption,
 			CustomHeader: cHeader,
@@ -1633,8 +1663,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
 		}
 		uploadedParts = append(uploadedParts, minio.CompletePart{
-			PartNumber: pInfo.PartNumber,
-			ETag:       pInfo.ETag,
+			PartNumber:     pInfo.PartNumber,
+			ETag:           pInfo.ETag,
+			ChecksumCRC32:  pInfo.ChecksumCRC32,
+			ChecksumCRC32C: pInfo.ChecksumCRC32C,
+			ChecksumSHA1:   pInfo.ChecksumSHA1,
+			ChecksumSHA256: pInfo.ChecksumSHA256,
 		})
 	}
 
@@ -2849,17 +2883,19 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 						ReplicationProxyRequest: "false",
 					},
 				})
+				sz := roi.Size
 				if err != nil {
 					if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
 						st.ReplicatedCount++
 					} else {
 						st.FailedCount++
 					}
+					sz = 0
 				} else {
 					st.ReplicatedCount++
 					st.ReplicatedSize += roi.Size
 				}
-				traceFn(err)
+				traceFn(sz, err)
 				select {
 				case <-ctx.Done():
 					return
@@ -2974,17 +3010,17 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 	return nil
 }
 
-func (s *replicationResyncer) trace(resyncID string, path string) func(err error) {
+func (s *replicationResyncer) trace(resyncID string, path string) func(sz int64, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(sz int64, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceReplicationResync) > 0 {
-			globalTrace.Publish(replicationResyncTrace(resyncID, startTime, duration, path, err))
+			globalTrace.Publish(replicationResyncTrace(resyncID, startTime, duration, path, err, sz))
 		}
 	}
 }
 
-func replicationResyncTrace(resyncID string, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
+func replicationResyncTrace(resyncID string, startTime time.Time, duration time.Duration, path string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -2998,6 +3034,7 @@ func replicationResyncTrace(resyncID string, startTime time.Time, duration time.
 		Duration:  duration,
 		Path:      path,
 		Error:     errStr,
+		Bytes:     sz,
 	}
 }
 

cmd/common-main.go

@@ -402,6 +402,29 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 		ctxt.certsDirSet = true
 	}
 
+	memAvailable := availableMemory()
+	if ctx.IsSet("memlimit") || ctx.GlobalIsSet("memlimit") {
+		memlimit := ctx.String("memlimit")
+		if memlimit == "" {
+			memlimit = ctx.GlobalString("memlimit")
+		}
+		mlimit, err := humanize.ParseBytes(memlimit)
+		if err != nil {
+			return err
+		}
+		if mlimit > memAvailable {
+			logger.Info("WARNING: maximum memory available (%s) smaller than specified --memlimit=%s, ignoring --memlimit value",
+				humanize.IBytes(memAvailable), memlimit)
+		}
+		ctxt.MemLimit = mlimit
+	} else {
+		ctxt.MemLimit = memAvailable
+	}
+
+	if memAvailable < ctxt.MemLimit {
+		ctxt.MemLimit = memAvailable
+	}
+
 	ctxt.FTP = ctx.StringSlice("ftp")
 	ctxt.SFTP = ctx.StringSlice("sftp")
 	ctxt.Interface = ctx.String("interface")

cmd/config-current.go

@@ -240,6 +240,11 @@ func initHelp() {
 			Description: "manage Browser HTTP specific features, such as Security headers, etc.",
 			Optional:    true,
 		},
+		config.HelpKV{
+			Key:         config.ILMSubSys,
+			Description: "manage ILM settings for expiration and transition workers",
+			Optional:    true,
+		},
 	}
 
 	if globalIsErasure {
@@ -288,6 +293,7 @@ func initHelp() {
 		config.DriveSubSys:          drive.HelpDrive,
 		config.CacheSubSys:          cache.Help,
 		config.BrowserSubSys:        browser.Help,
+		config.ILMSubSys:            ilm.Help,
 	}
 
 	config.RegisterHelpSubSys(helpMap)

cmd/erasure-healing.go

@@ -1065,6 +1065,7 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object strin
 		if result != nil {
 			tr.Custom["version-id"] = result.VersionID
 			tr.Custom["disks"] = strconv.Itoa(result.DiskCount)
+			tr.Bytes = result.ObjectSize
 		}
 	}
 	if err != nil {

cmd/erasure-multipart.go

@@ -1296,7 +1296,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	if checksumType.IsSet() {
 		checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
-		cs := hash.NewChecksumFromData(checksumType, checksumCombined)
+		var cs *hash.Checksum
+		cs = hash.NewChecksumFromData(checksumType, checksumCombined)
 		fi.Checksum = cs.AppendTo(nil, checksumCombined)
 		if opts.EncryptFn != nil {
 			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)

cmd/erasure-server-pool-decom.go

@@ -836,7 +836,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				if filterLifecycle(bi.Name, version.Name, version) {
 					expired++
 					decommissioned++
-					stopFn(errors.New("ILM expired object/version will be skipped"))
+					stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
 					continue
 				}
 
@@ -846,7 +846,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				remainingVersions := len(fivs.Versions) - expired
 				if version.Deleted && remainingVersions == 1 {
 					decommissioned++
-					stopFn(errors.New("DELETE marked object with no other non-current versions will be skipped"))
+					stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
 					continue
 				}
 
@@ -877,7 +877,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							err = nil
 						}
 					}
-					stopFn(err)
+					stopFn(version.Size, err)
 					if err != nil {
 						decomLogIf(ctx, err)
 						failure = true
@@ -902,12 +902,12 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							MTime:       version.ModTime,
 							UserDefined: version.Metadata,
 						}); err != nil {
-							stopFn(err)
+							stopFn(version.Size, err)
 							failure = true
 							decomLogIf(ctx, err)
 							continue
 						}
-						stopFn(nil)
+						stopFn(version.Size, nil)
 						failure = false
 						break
 					}
@@ -925,14 +925,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
-						stopFn(nil)
+						stopFn(version.Size, nil)
 						break
 					}
 					if err != nil && !ignore {
 						// if usage-cache.bin is not readable log and ignore it.
 						if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) {
 							ignore = true
-							stopFn(err)
+							stopFn(version.Size, err)
 							decomLogIf(ctx, err)
 							break
 						}
@@ -940,16 +940,16 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if err != nil {
 						failure = true
 						decomLogIf(ctx, err)
-						stopFn(err)
+						stopFn(version.Size, err)
 						continue
 					}
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
-						stopFn(err)
+						stopFn(version.Size, err)
 						failure = true
 						decomLogIf(ctx, err)
 						continue
 					}
-					stopFn(nil)
+					stopFn(version.Size, nil)
 					failure = false
 					break
 				}
@@ -977,7 +977,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						NoAuditLog:         true,
 					},
 				)
-				stopFn(err)
+				stopFn(0, err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
 					decomLogIf(ctx, err)
@@ -1038,7 +1038,7 @@ const (
 	decomMetricDecommissionRemoveObject
 )
 
-func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
+func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -1051,15 +1051,16 @@ func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.D
 		Duration:  duration,
 		Path:      path,
 		Error:     errStr,
+		Bytes:     sz,
 	}
 }
 
-func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err error) {
+func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(z int64, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(sz int64, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceDecommission) > 0 {
-			globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err))
+			globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err, sz))
 		}
 	}
 }
@@ -1092,10 +1093,10 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		}
 		stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name)
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
-			stopFn(err)
+			stopFn(0, err)
 			return err
 		}
-		stopFn(nil)
+		stopFn(0, nil)
 
 		z.poolMetaMutex.Lock()
 		if z.poolMeta.BucketDone(idx, bucket) {

cmd/erasure-server-pool-rebalance.go

@@ -427,7 +427,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 
 			stopFn := globalRebalanceMetrics.log(rebalanceMetricSaveMetadata, poolIdx, traceMsg)
 			err := z.saveRebalanceStats(GlobalContext, poolIdx, rebalSaveStats)
-			stopFn(err)
+			stopFn(0, err)
 			rebalanceLogIf(GlobalContext, err)
 
 			if quit {
@@ -456,7 +456,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 
 		stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBucket, poolIdx, bucket)
 		if err = z.rebalanceBucket(ctx, bucket, poolIdx); err != nil {
-			stopFn(err)
+			stopFn(0, err)
 			if errors.Is(err, errServerNotInitialized) || errors.Is(err, errBucketMetadataNotInitialized) {
 				continue
 			}
@@ -464,7 +464,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 			doneCh <- err
 			return
 		}
-		stopFn(nil)
+		stopFn(0, nil)
 		z.bucketRebalanceDone(bucket, poolIdx)
 	}
 
@@ -692,24 +692,24 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
-						stopFn(nil)
+						stopFn(0, nil)
 						break
 					}
 					if err != nil {
 						failure = true
 						rebalanceLogIf(ctx, err)
-						stopFn(err)
+						stopFn(0, err)
 						continue
 					}
 
 					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
 						failure = true
 						rebalanceLogIf(ctx, err)
-						stopFn(err)
+						stopFn(version.Size, err)
 						continue
 					}
 
-					stopFn(nil)
+					stopFn(version.Size, nil)
 					failure = false
 					break
 				}
@@ -735,7 +735,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						NoAuditLog:         true,
 					},
 				)
-				stopFn(err)
+				stopFn(0, err)
 				auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
 				if err != nil {
 					rebalanceLogIf(ctx, err)
@@ -935,7 +935,7 @@ func (z *erasureServerPools) StartRebalance() {
 		go func(idx int) {
 			stopfn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBuckets, idx)
 			err := z.rebalanceBuckets(ctx, idx)
-			stopfn(err)
+			stopfn(0, err)
 		}(poolIdx)
 	}
 }
@@ -975,7 +975,7 @@ const (
 	rebalanceMetricSaveMetadata
 )
 
-func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string) madmin.TraceInfo {
+func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -988,15 +988,16 @@ func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duratio
 		Duration:  duration,
 		Path:      path,
 		Error:     errStr,
+		Bytes:     sz,
 	}
 }
 
-func (p *rebalanceMetrics) log(r rebalanceMetric, poolIdx int, paths ...string) func(err error) {
+func (p *rebalanceMetrics) log(r rebalanceMetric, poolIdx int, paths ...string) func(sz int64, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(sz int64, err error) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceRebalance) > 0 {
-			globalTrace.Publish(rebalanceTrace(r, poolIdx, startTime, duration, err, strings.Join(paths, " ")))
+			globalTrace.Publish(rebalanceTrace(r, poolIdx, startTime, duration, err, strings.Join(paths, " "), sz))
 		}
 	}
 }

cmd/erasure-server-pool.go

@@ -104,7 +104,11 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
 	buffers := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
-	buffers.Populate()
+	if n >= 16384 {
+		// pre-populate buffers only n >= 16384 which is (32Gi/2Mi)
+		// for all setups smaller than this avoid pre-alloc.
+		buffers.Populate()
+	}
 	globalBytePoolCap.Store(buffers)
 
 	var localDrives []StorageAPI
@@ -165,6 +169,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	if !globalIsDistErasure {
 		globalLocalDrivesMu.Lock()
 		globalLocalDrives = localDrives
+		for _, drive := range localDrives {
+			globalLocalDrivesMap[drive.Endpoint().String()] = drive
+		}
 		globalLocalDrivesMu.Unlock()
 	}
 
@@ -1535,7 +1542,7 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 		return loi, nil
 	}
 	ri := logger.GetReqInfo(ctx)
-	hadoop := ri != nil && strings.Contains(ri.UserAgent, `Hadoop `) && strings.Contains(ri.UserAgent, "scala/")
+	hadoop := ri != nil && strings.Contains(ri.UserAgent, "Hadoop ") && strings.Contains(ri.UserAgent, "scala/")
 	matches := func() bool {
 		if prefix == "" {
 			return false
@@ -1600,7 +1607,7 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 		// }
 		if matches() {
 			objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true})
-			if err == nil {
+			if err == nil || objInfo.IsLatest && objInfo.DeleteMarker {
 				if opts.Lifecycle != nil {
 					evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
 					if evt.Action.Delete() {
@@ -1626,41 +1633,29 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 		// and throw a 404 exception. This is especially a problem for spark jobs overwriting the same partition
 		// repeatedly. This workaround recursively lists the top 3 entries including delete markers to reflect the
 		// correct state of the directory in the list results.
-		opts.Recursive = true
-		opts.InclDeleted = true
-		opts.Limit = maxKeys + 1
-		li, err := listFn(ctx, opts, opts.Limit)
-		if err == nil {
-			switch {
-			case len(li.Objects) == 0 && len(li.Prefixes) == 0:
-				return loi, nil
-			case len(li.Objects) > 0 || len(li.Prefixes) > 0:
-				var o ObjectInfo
-				var pfx string
-				if len(li.Objects) > 0 {
-					o = li.Objects[0]
-					p := strings.TrimPrefix(o.Name, opts.Prefix)
-					if p != "" {
-						sidx := strings.Index(p, "/")
-						if sidx > 0 {
-							pfx = p[:sidx]
-						}
-					}
-				}
-				switch {
-				case o.DeleteMarker:
-					loi.Objects = append(loi.Objects, ObjectInfo{Bucket: bucket, IsDir: true, Name: prefix})
-					return loi, nil
-				case len(li.Objects) >= 1:
-					loi.Objects = append(loi.Objects, o)
-					if pfx != "" {
-						loi.Prefixes = append(loi.Prefixes, path.Join(opts.Prefix, pfx))
-					}
-				case len(li.Prefixes) > 0:
-					loi.Prefixes = append(loi.Prefixes, li.Prefixes...)
-				}
+		if strings.HasSuffix(opts.Prefix, SlashSeparator) {
+			li, err := listFn(ctx, opts, maxKeys)
+			if err != nil {
+				return loi, err
 			}
-			return loi, nil
+			if len(li.Objects) == 0 {
+				prefixes := li.Prefixes[:0]
+				for _, prefix := range li.Prefixes {
+					objInfo, _ := z.GetObjectInfo(ctx, bucket, pathJoin(prefix, "_SUCCESS"), ObjectOptions{NoLock: true})
+					if objInfo.IsLatest && objInfo.DeleteMarker {
+						continue
+					}
+					prefixes = append(prefixes, prefix)
+				}
+				if len(prefixes) > 0 {
+					objInfo, _ := z.GetObjectInfo(ctx, bucket, pathJoin(opts.Prefix, "_SUCCESS"), ObjectOptions{NoLock: true})
+					if objInfo.IsLatest && objInfo.DeleteMarker {
+						return loi, nil
+					}
+				}
+				li.Prefixes = prefixes
+			}
+			return li, nil
 		}
 	}
 

cmd/ftp-server-driver.go

@@ -105,7 +105,7 @@ type ftpMetrics struct{}
 
 var globalFtpMetrics ftpMetrics
 
-func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error) madmin.TraceInfo {
+func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -114,25 +114,33 @@ func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err e
 		TraceType: madmin.TraceFTP,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
-		FuncName:  fmt.Sprintf("ftp USER=%s COMMAND=%s PARAM=%s ISLOGIN=%t, Source=%s", s.Sess.LoginUser(), s.Cmd, s.Param, s.Sess.IsLogin(), source),
+		FuncName:  fmt.Sprintf(s.Cmd),
 		Duration:  time.Since(startTime),
 		Path:      objPath,
 		Error:     errStr,
+		Bytes:     sz,
+		Custom: map[string]string{
+			"user":   s.Sess.LoginUser(),
+			"cmd":    s.Cmd,
+			"param":  s.Param,
+			"login":  fmt.Sprintf("%t", s.Sess.IsLogin()),
+			"source": source,
+		},
 	}
 }
 
-func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(err error) {
+func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(sz int64, err error) {
 	startTime := time.Now()
 	source := getSource(2)
-	return func(err error) {
-		globalTrace.Publish(ftpTrace(s, startTime, source, strings.Join(paths, " "), err))
+	return func(sz int64, err error) {
+		globalTrace.Publish(ftpTrace(s, startTime, source, strings.Join(paths, " "), err, sz))
 	}
 }
 
 // Stat implements ftpDriver
 func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo, err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	if objPath == SlashSeparator {
 		return &minioFileInfo{
@@ -190,7 +198,7 @@ func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo,
 // ListDir implements ftpDriver
 func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func(os.FileInfo) error) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	clnt, err := driver.getMinIOClient(ctx)
 	if err != nil {
@@ -252,7 +260,7 @@ func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func
 
 func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string) (ok bool, err error) {
 	stopFn := globalFtpMetrics.log(c, username)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	if globalIAMSys.LDAPConfig.Enabled() {
 		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), username)
@@ -376,7 +384,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 // DeleteDir implements ftpDriver
 func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
@@ -426,7 +434,7 @@ func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error)
 // DeleteFile implements ftpDriver
 func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
@@ -444,7 +452,7 @@ func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error
 // Rename implements ftpDriver
 func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, fromObjPath, toObjPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	return NotImplemented{}
 }
@@ -452,7 +460,7 @@ func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath
 // MakeDir implements ftpDriver
 func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
@@ -479,7 +487,7 @@ func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) {
 // GetFile implements ftpDriver
 func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64) (n int64, rc io.ReadCloser, err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(n, err)
 
 	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
@@ -511,14 +519,14 @@ func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64)
 	if err != nil {
 		return 0, nil, err
 	}
-
-	return info.Size - offset, obj, nil
+	n = info.Size - offset
+	return n, obj, nil
 }
 
 // PutFile implements ftpDriver
 func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reader, offset int64) (n int64, err error) {
 	stopFn := globalFtpMetrics.log(ctx, objPath)
-	defer stopFn(err)
+	defer stopFn(n, err)
 
 	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
@@ -539,5 +547,6 @@ func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reade
 		ContentType:          mimedb.TypeByExtension(path.Ext(object)),
 		DisableContentSha256: true,
 	})
-	return info.Size, err
+	n = info.Size
+	return n, err
 }

cmd/generic-handlers.go

@@ -601,13 +601,6 @@ func setUploadForwardingMiddleware(h http.Handler) http.Handler {
 				h.ServeHTTP(w, r)
 				return
 			}
-			// forward request to peer handling this upload
-			if globalBucketTargetSys.isOffline(remote.EndpointURL) {
-				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
-				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrReplicationRemoteConnectionError), r.URL)
-				return
-			}
-
 			r.URL.Scheme = remote.EndpointURL.Scheme
 			r.URL.Host = remote.EndpointURL.Host
 			// Make sure we remove any existing headers before

cmd/global-heal.go

@@ -530,7 +530,16 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	tracker.setObject("")
 	tracker.setBucket("")
 
-	return retErr
+	if retErr != nil {
+		return retErr
+	}
+
+	// Last sanity check
+	if len(tracker.QueuedBuckets) > 0 {
+		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
+	}
+
+	return nil
 }
 
 func healBucket(bucket string, scan madmin.HealScanMode) error {

cmd/globals.go

@@ -160,6 +160,8 @@ type serverCtxt struct {
 	FTP  []string
 	SFTP []string
 
+	MemLimit uint64
+
 	UserTimeout         time.Duration
 	ShutdownTimeout     time.Duration
 	IdleTimeout         time.Duration

cmd/handler-api.go

@@ -91,11 +91,11 @@ func availableMemory() (available uint64) {
 	available = 2048 * blockSizeV2 * 2 // Default to 4 GiB when we can't find the limits.
 
 	if runtime.GOOS == "linux" {
-		// Useful in container mode
+		// Honor cgroup limits if set.
 		limit := cgroupMemLimit()
 		if limit > 0 {
-			// A valid value is found, return its 75%
-			available = (limit * 3) / 4
+			// A valid value is found, return its 90%
+			available = (limit * 9) / 10
 			return
 		}
 	} // for all other platforms limits are based on virtual memory.
@@ -104,8 +104,9 @@ func availableMemory() (available uint64) {
 	if err != nil {
 		return
 	}
-	// A valid value is available return its 75%
-	available = (memStats.Available * 3) / 4
+
+	// A valid value is available return its 90%
+	available = (memStats.Available * 9) / 10
 	return
 }
 
@@ -129,7 +130,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 		maxSetDrives := slices.Max(setDriveCounts)
 
 		// Returns 75% of max memory allowed
-		maxMem := availableMemory()
+		maxMem := globalServerCtxt.MemLimit
 
 		// max requests per node is calculated as
 		// total_ram / ram_per_request

cmd/http-tracer.go

@@ -142,6 +142,7 @@ func httpTracerMiddleware(h http.Handler) http.Handler {
 			Time:      reqStartTime,
 			Duration:  reqEndTime.Sub(respRecorder.StartTime),
 			Path:      reqPath,
+			Bytes:     int64(inputBytes + respRecorder.Size()),
 			HTTP: &madmin.TraceHTTPStats{
 				ReqInfo: madmin.TraceRequestInfo{
 					Time:     reqStartTime,

cmd/iam-store.go

@@ -2371,7 +2371,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 			return updatedAt, err
 		}
 
-		if len(policyBuf) > 2048 {
+		if len(policyBuf) > maxSVCSessionPolicySize {
 			return updatedAt, errSessionPolicyTooLarge
 		}
 

cmd/iam.go

@@ -78,6 +78,10 @@ const (
 	inheritedPolicyType = "inherited-policy"
 )
 
+const (
+	maxSVCSessionPolicySize = 4096
+)
+
 // IAMSys - config system.
 type IAMSys struct {
 	// Need to keep them here to keep alignment - ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
@@ -977,7 +981,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		if err != nil {
 			return auth.Credentials{}, time.Time{}, err
 		}
-		if len(policyBuf) > 2048 {
+		if len(policyBuf) > maxSVCSessionPolicySize {
 			return auth.Credentials{}, time.Time{}, errSessionPolicyTooLarge
 		}
 	}
@@ -1982,20 +1986,22 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		}
 		isGroup = false
 	} else {
-		if isAttach {
-			var underBaseDN bool
-			if dnResult, underBaseDN, err = sys.LDAPConfig.GetValidatedGroupDN(nil, r.Group); err != nil {
-				iamLogIf(ctx, err)
-				return
-			} else if dnResult == nil || !underBaseDN {
+		var underBaseDN bool
+		if dnResult, underBaseDN, err = sys.LDAPConfig.GetValidatedGroupDN(nil, r.Group); err != nil {
+			iamLogIf(ctx, err)
+			return
+		}
+		if dnResult == nil || !underBaseDN {
+			if !isAttach {
+				dn = r.Group
+			} else {
 				err = errNoSuchGroup
 				return
 			}
+		} else {
 			// We use the group DN returned by the LDAP server (this may not
 			// equal the input group name, but we assume it is canonical).
 			dn = dnResult.NormDN
-		} else {
-			dn = r.Group
 		}
 		isGroup = true
 	}

cmd/logging.go

@@ -8,6 +8,10 @@ import (
 	"github.com/minio/minio/internal/logger"
 )
 
+func proxyLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "proxy", err, errKind...)
+}
+
 func replLogIf(ctx context.Context, err error, errKind ...interface{}) {
 	logger.LogIf(ctx, "replication", err, errKind...)
 }

cmd/metacache-walk.go

@@ -405,7 +405,7 @@ func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions,
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.WalkDir(ctx, opts, wr)
 }

cmd/metrics-realtime.go

@@ -132,6 +132,15 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 			m.Aggregated.CPU.LoadStat = loadStat
 		}
 	}
+	if types.Contains(madmin.MetricsRPC) {
+		gr := globalGrid.Load()
+		if gr == nil {
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: Grid not initialized", byHostName))
+		} else {
+			stats := gr.ConnStats()
+			m.Aggregated.RPC = &stats
+		}
+	}
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.

cmd/metrics-v2.go

@@ -273,13 +273,10 @@ const (
 	vmemory          = "virtual_memory_bytes"
 	cpu              = "cpu_total_seconds"
 
-	expiryPendingTasks           MetricName = "expiry_pending_tasks"
 	expiryMissedTasks            MetricName = "expiry_missed_tasks"
 	expiryMissedFreeVersions     MetricName = "expiry_missed_freeversions"
 	expiryMissedTierJournalTasks MetricName = "expiry_missed_tierjournal_tasks"
 	expiryNumWorkers             MetricName = "expiry_num_workers"
-	transitionPendingTasks       MetricName = "transition_pending_tasks"
-	transitionActiveTasks        MetricName = "transition_active_tasks"
 	transitionMissedTasks        MetricName = "transition_missed_immediate_tasks"
 
 	transitionedBytes    MetricName = "transitioned_bytes"

cmd/metrics-v3-ilm.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2024 MinIO, Inc.
+//
+// # This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+)
+
+const (
+	expiryPendingTasks             = "expiry_pending_tasks"
+	transitionActiveTasks          = "transition_active_tasks"
+	transitionPendingTasks         = "transition_pending_tasks"
+	transitionMissedImmediateTasks = "transition_missed_immediate_tasks"
+	versionsScanned                = "versions_scanned"
+)
+
+var (
+	ilmExpiryPendingTasksMD             = NewGaugeMD(expiryPendingTasks, "Number of pending ILM expiry tasks in the queue")
+	ilmTransitionActiveTasksMD          = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks")
+	ilmTransitionPendingTasksMD         = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue")
+	ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks")
+	ilmVersionsScannedMD                = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start")
+)
+
+// loadILMMetrics - `MetricsLoaderFn` for ILM metrics.
+func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
+	if globalExpiryState != nil {
+		m.Set(expiryPendingTasks, float64(globalExpiryState.PendingTasks()))
+	}
+	if globalTransitionState != nil {
+		m.Set(transitionActiveTasks, float64(globalTransitionState.ActiveTasks()))
+		m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks()))
+		m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks()))
+	}
+	m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM)))
+
+	return nil
+}

cmd/metrics-v3.go

@@ -54,6 +54,7 @@ const (
 	clusterIAMCollectorPath          collectorPath = "/cluster/iam"
 	clusterConfigCollectorPath       collectorPath = "/cluster/config"
 
+	ilmCollectorPath           collectorPath = "/ilm"
 	auditCollectorPath         collectorPath = "/audit"
 	loggerWebhookCollectorPath collectorPath = "/logger/webhook"
 	replicationCollectorPath   collectorPath = "/replication"
@@ -382,6 +383,17 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadAuditMetrics,
 	)
 
+	ilmMG := NewMetricsGroup(ilmCollectorPath,
+		[]MetricDescriptor{
+			ilmExpiryPendingTasksMD,
+			ilmTransitionActiveTasksMD,
+			ilmTransitionPendingTasksMD,
+			ilmTransitionMissedImmediateTasksMD,
+			ilmVersionsScannedMD,
+		},
+		loadILMMetrics,
+	)
+
 	allMetricGroups := []*MetricsGroup{
 		apiRequestsMG,
 		bucketAPIMG,
@@ -402,6 +414,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		clusterReplicationMG,
 		clusterConfigMG,
 
+		ilmMG,
 		scannerMG,
 		auditMG,
 		loggerWebhookMG,

cmd/object-api-utils.go

@@ -1206,7 +1206,20 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 	}
 
 	if nDisks < len(di)/2 || nDisks <= 0 {
-		return false, fmt.Errorf("not enough online disks to calculate the available space, expected (%d)/(%d)", (len(di)/2)+1, nDisks)
+		var errs []error
+		for index, disk := range di {
+			switch {
+			case disk == nil:
+				errs = append(errs, fmt.Errorf("disk[%d]: offline", index))
+			case disk.Error != "":
+				errs = append(errs, fmt.Errorf("disk %s: %s", disk.Endpoint, disk.Error))
+			case disk.Total == 0:
+				errs = append(errs, fmt.Errorf("disk %s: total is zero", disk.Endpoint))
+			}
+		}
+		// Log disk errors.
+		peersLogIf(context.Background(), errors.Join(errs...))
+		return false, fmt.Errorf("not enough online disks to calculate the available space, need %d, found %d", (len(di)/2)+1, nDisks)
 	}
 
 	// Check we have enough on each disk, ignoring diskFillFraction.

cmd/object-handlers_test.go

@@ -2914,7 +2914,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	s3MD5 := getCompleteMultipartMD5(inputParts[3].parts)
 
 	// generating the response body content for the success case.
-	successResponse := generateCompleteMultpartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5})
+	successResponse := generateCompleteMultipartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5})
 	encodedSuccessResponse := encodeResponse(successResponse)
 
 	ctx := context.Background()

cmd/object-multipart-handlers.go

@@ -773,6 +773,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
+	_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
 	var objectEncryptionKey crypto.ObjectKey
 	if isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) && !replicationStatus {
@@ -795,7 +796,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			}
 		}
 
-		_, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest]
 		if !(sourceReplReq && crypto.SSEC.IsEncrypted(mi.UserDefined)) {
 			// Calculating object encryption key
 			key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined)
@@ -847,6 +847,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 	opts.IndexCB = idxCb
 
+	opts.ReplicationRequest = sourceReplReq
 	putObjectPart := objectAPI.PutObjectPart
 
 	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, pReader, opts)
@@ -1040,7 +1041,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	// Get object location.
 	location := getObjectLocation(r, globalDomainNames, bucket, object)
 	// Generate complete multipart response.
-	response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo)
+	response := generateCompleteMultipartUploadResponse(bucket, object, location, objInfo)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// Write success response.

cmd/server-main.go

@@ -414,11 +414,12 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 	setGlobalInternodeInterface(ctxt.Interface)
 
 	globalTCPOptions = xhttp.TCPOptions{
-		UserTimeout:    int(ctxt.UserTimeout.Milliseconds()),
-		DriveOPTimeout: globalDriveConfig.GetOPTimeout,
-		Interface:      ctxt.Interface,
-		SendBufSize:    ctxt.SendBufSize,
-		RecvBufSize:    ctxt.RecvBufSize,
+		UserTimeout: int(ctxt.UserTimeout.Milliseconds()),
+		// FIXME: Bring this back when we have valid way to handle deadlines
+		//		DriveOPTimeout: globalDriveConfig.GetOPTimeout,
+		Interface:   ctxt.Interface,
+		SendBufSize: ctxt.SendBufSize,
+		RecvBufSize: ctxt.RecvBufSize,
 	}
 
 	// allow transport to be HTTP/1.1 for proxying.
@@ -430,7 +431,7 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
 		RoundTripper: globalRemoteTargetTransport,
 		Logger: func(err error) {
 			if err != nil && !errors.Is(err, context.Canceled) {
-				replLogIf(GlobalContext, err)
+				proxyLogIf(GlobalContext, err)
 			}
 		},
 	})
@@ -834,7 +835,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Set system resources to maximum.
 	bootstrapTrace("setMaxResources", func() {
-		_ = setMaxResources(ctx)
+		_ = setMaxResources(globalServerCtxt)
 	})
 
 	// Verify kernel release and version.

cmd/server-rlimit.go

@@ -22,7 +22,6 @@ import (
 	"runtime/debug"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/cli"
 	"github.com/minio/madmin-go/v3/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/sys"
@@ -45,7 +44,7 @@ func oldLinux() bool {
 	return currentKernel < kernel.Version(4, 0, 0)
 }
 
-func setMaxResources(ctx *cli.Context) (err error) {
+func setMaxResources(ctx serverCtxt) (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, err := sys.GetMaxThreads()
 	if err == nil {
@@ -72,32 +71,28 @@ func setMaxResources(ctx *cli.Context) (err error) {
 		return err
 	}
 
-	// Set max memory limit as current memory limit.
-	if _, maxLimit, err = sys.GetMaxMemoryLimit(); err != nil {
+	_, vssLimit, err := sys.GetMaxMemoryLimit()
+	if err != nil {
 		return err
 	}
 
-	// set debug memory limit instead of GOMEMLIMIT env
-	_ = setDebugMemoryLimit(ctx)
-
-	err = sys.SetMaxMemoryLimit(maxLimit, maxLimit)
-	return err
-}
-
-func setDebugMemoryLimit(ctx *cli.Context) error {
-	if ctx == nil {
-		return nil
+	if vssLimit > 0 && vssLimit < humanize.GiByte {
+		logger.Info("WARNING: maximum virtual memory limit (%s) is too small for 'go runtime', please consider setting `ulimit -v` to unlimited",
+			humanize.IBytes(vssLimit))
 	}
-	if ctx.IsSet("memlimit") {
-		memlimit := ctx.String("memlimit")
-		if memlimit == "" {
-			memlimit = ctx.GlobalString("memlimit")
-		}
-		mlimit, err := humanize.ParseBytes(memlimit)
-		if err != nil {
-			return err
-		}
-		debug.SetMemoryLimit(int64(mlimit))
+
+	if ctx.MemLimit > 0 {
+		maxLimit = ctx.MemLimit
 	}
+
+	if maxLimit > 0 {
+		debug.SetMemoryLimit(int64(maxLimit))
+	}
+
+	// Do not use RLIMIT_AS as that is not useful and at times on systems < 4Gi
+	// this can crash the Go runtime if the value is smaller refer
+	// - https://github.com/golang/go/issues/38010
+	// - https://github.com/golang/go/issues/43699
+	// So do not add `sys.SetMaxMemoryLimit()` this is not useful for any practical purposes.
 	return nil
 }

cmd/sftp-server-driver.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,13 +32,16 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/minio/minio/internal/auth"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/pkg/v3/mimedb"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
 
+// Maximum write offset for incoming SFTP blocks.
+// Set to 100MiB to prevent hostile DOS attacks.
+const ftpMaxWriteOffset = 100 << 20
+
 type sftpDriver struct {
 	permissions *ssh.Permissions
 	endpoint    string
@@ -49,7 +52,7 @@ type sftpMetrics struct{}
 
 var globalSftpMetrics sftpMetrics
 
-func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string, err error) madmin.TraceInfo {
+func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string, err error, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -58,18 +61,25 @@ func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string,
 		TraceType: madmin.TraceFTP,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
-		FuncName:  fmt.Sprintf("sftp USER=%s COMMAND=%s PARAM=%s, Source=%s", user, s.Method, s.Filepath, source),
+		FuncName:  s.Method,
 		Duration:  time.Since(startTime),
 		Path:      s.Filepath,
 		Error:     errStr,
+		Bytes:     sz,
+		Custom: map[string]string{
+			"user":   user,
+			"cmd":    s.Method,
+			"param":  s.Filepath,
+			"source": source,
+		},
 	}
 }
 
-func (m *sftpMetrics) log(s *sftp.Request, user string) func(err error) {
+func (m *sftpMetrics) log(s *sftp.Request, user string) func(sz int64, err error) {
 	startTime := time.Now()
 	source := getSource(2)
-	return func(err error) {
-		globalTrace.Publish(sftpTrace(s, startTime, source, user, err))
+	return func(sz int64, err error) {
+		globalTrace.Publish(sftpTrace(s, startTime, source, user, err, sz))
 	}
 }
 
@@ -90,108 +100,26 @@ func NewSFTPDriver(perms *ssh.Permissions) sftp.Handlers {
 }
 
 func (f *sftpDriver) getMinIOClient() (*minio.Client, error) {
-	ui, ok := globalIAMSys.GetUser(context.Background(), f.AccessKey())
-	if !ok && !globalIAMSys.LDAPConfig.Enabled() {
-		return nil, errNoSuchUser
-	}
-	if !ok && globalIAMSys.LDAPConfig.Enabled() {
-		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), f.AccessKey())
-		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
-			return nil, err
-		}
-		var mcreds *credentials.Credentials
-		if errors.Is(err, errNoSuchServiceAccount) {
-			lookupResult, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(f.AccessKey())
-			if err != nil {
-				return nil, err
-			}
-			expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
-			if err != nil {
-				return nil, err
-			}
-			claims := make(map[string]interface{})
-			claims[expClaim] = UTCNow().Add(expiryDur).Unix()
-			for k, v := range f.permissions.CriticalOptions {
-				claims[k] = v
-			}
-
-			// Set LDAP claims.
-			claims[ldapUserN] = f.AccessKey()
-			claims[ldapUser] = lookupResult.NormDN
-			// Add LDAP attributes that were looked up into the claims.
-			for attribKey, attribValue := range lookupResult.Attributes {
-				claims[ldapAttribPrefix+attribKey] = attribValue
-			}
-
-			cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
-			if err != nil {
-				return nil, err
-			}
-
-			// Set the parent of the temporary access key, this is useful
-			// in obtaining service accounts by this cred.
-			cred.ParentUser = lookupResult.NormDN
-
-			// Set this value to LDAP groups, LDAP user can be part
-			// of large number of groups
-			cred.Groups = targetGroups
-
-			// Set the newly generated credentials, policyName is empty on purpose
-			// LDAP policies are applied automatically using their ldapUser, ldapGroups
-			// mapping.
-			updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
-			if err != nil {
-				return nil, err
-			}
-
-			// Call hook for site replication.
-			replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
-				Type: madmin.SRIAMItemSTSAcc,
-				STSCredential: &madmin.SRSTSCredential{
-					AccessKey:    cred.AccessKey,
-					SecretKey:    cred.SecretKey,
-					SessionToken: cred.SessionToken,
-					ParentUser:   cred.ParentUser,
-				},
-				UpdatedAt: updatedAt,
-			}))
-
-			mcreds = credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken)
-		} else {
-			mcreds = credentials.NewStaticV4(sa.Credentials.AccessKey, sa.Credentials.SecretKey, "")
-		}
-
-		return minio.New(f.endpoint, &minio.Options{
-			Creds:     mcreds,
-			Secure:    globalIsTLS,
-			Transport: globalRemoteFTPClientTransport,
-		})
-	}
-
-	// ok == true - at this point
-
-	if ui.Credentials.IsTemp() {
-		// Temporary credentials are not allowed.
-		return nil, errAuthentication
-	}
-
+	mcreds := credentials.NewStaticV4(
+		f.permissions.CriticalOptions["AccessKey"],
+		f.permissions.CriticalOptions["SecretKey"],
+		f.permissions.CriticalOptions["SessionToken"],
+	)
 	return minio.New(f.endpoint, &minio.Options{
-		Creds:     credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""),
+		Creds:     mcreds,
 		Secure:    globalIsTLS,
 		Transport: globalRemoteFTPClientTransport,
 	})
 }
 
 func (f *sftpDriver) AccessKey() string {
-	if _, ok := f.permissions.CriticalOptions["accessKey"]; !ok {
-		return f.permissions.CriticalOptions[ldapUserN]
-	}
-	return f.permissions.CriticalOptions["accessKey"]
+	return f.permissions.CriticalOptions["AccessKey"]
 }
 
 func (f *sftpDriver) Fileread(r *sftp.Request) (ra io.ReaderAt, err error) {
+	// This is not timing the actual read operation, but the time it takes to prepare the reader.
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	flags := r.Pflags()
 	if !flags.Read {
@@ -269,6 +197,9 @@ func (w *writerAt) WriteAt(b []byte, offset int64) (n int, err error) {
 		n, err = w.w.Write(b)
 		w.nextOffset += int64(n)
 	} else {
+		if offset > w.nextOffset+ftpMaxWriteOffset {
+			return 0, fmt.Errorf("write offset %d is too far ahead of next offset %d", offset, w.nextOffset)
+		}
 		w.buffer[offset] = make([]byte, len(b))
 		copy(w.buffer[offset], b)
 		n = len(b)
@@ -294,7 +225,12 @@ again:
 
 func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer func() {
+		if err != nil {
+			// If there is an error, we never started the goroutine.
+			stopFn(0, err)
+		}
+	}()
 
 	flags := r.Pflags()
 	if !flags.Write {
@@ -329,10 +265,11 @@ func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
 	}
 	wa.wg.Add(1)
 	go func() {
-		_, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{
+		oi, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{
 			ContentType:          mimedb.TypeByExtension(path.Ext(object)),
 			DisableContentSha256: true,
 		})
+		stopFn(oi.Size, err)
 		pr.CloseWithError(err)
 		wa.wg.Done()
 	}()
@@ -341,7 +278,7 @@ func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
 
 func (f *sftpDriver) Filecmd(r *sftp.Request) (err error) {
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	clnt, err := f.getMinIOClient()
 	if err != nil {
@@ -437,7 +374,7 @@ func (f listerAt) ListAt(ls []os.FileInfo, offset int64) (int, error) {
 
 func (f *sftpDriver) Filelist(r *sftp.Request) (la sftp.ListerAt, err error) {
 	stopFn := globalSftpMetrics.log(r, f.AccessKey())
-	defer stopFn(err)
+	defer stopFn(0, err)
 
 	clnt, err := f.getMinIOClient()
 	if err != nil {

cmd/sftp-server.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"crypto/subtle"
 	"errors"
@@ -29,18 +28,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/logger"
+	xldap "github.com/minio/pkg/v3/ldap"
 	xsftp "github.com/minio/pkg/v3/sftp"
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
 )
 
-type sftpLogger struct{}
-
-func (s *sftpLogger) Info(tag xsftp.LogType, msg string) {
-	logger.Info(msg)
-}
-
 const (
 	kexAlgoDH1SHA1                = "diffie-hellman-group1-sha1"
 	kexAlgoDH14SHA1               = "diffie-hellman-group14-sha1"
@@ -58,6 +54,16 @@ const (
 	tripledescbcID     = "3des-cbc"
 )
 
+var (
+	errSFTPPublicKeyBadFormat = errors.New("the public key provided could not be parsed")
+	errSFTPUserHasNoPolicies  = errors.New("no policies present on this account")
+	errSFTPLDAPNotEnabled     = errors.New("ldap authentication is not enabled")
+)
+
+// if the sftp parameter --trusted-user-ca-key is set, then
+// the final form of the key file will be set as this variable.
+var caPublicKey ssh.PublicKey
+
 // https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=46
 // preferredKexAlgos specifies the default preference for key-exchange
 // algorithms in preference order. The diffie-hellman-group16-sha512 algorithm
@@ -120,6 +126,226 @@ var supportedMACs = []string{
 	"hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1", "hmac-sha1-96",
 }
 
+func sshPubKeyAuth(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
+	return authenticateSSHConnection(c, key, nil)
+}
+
+func sshPasswordAuth(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+	return authenticateSSHConnection(c, nil, pass)
+}
+
+func authenticateSSHConnection(c ssh.ConnMetadata, key ssh.PublicKey, pass []byte) (*ssh.Permissions, error) {
+	user, found := strings.CutSuffix(c.User(), "=ldap")
+	if found {
+		if !globalIAMSys.LDAPConfig.Enabled() {
+			return nil, errSFTPLDAPNotEnabled
+		}
+		return processLDAPAuthentication(key, pass, user)
+	}
+
+	user, found = strings.CutSuffix(c.User(), "=svc")
+	if found {
+		goto internalAuth
+	}
+
+	if globalIAMSys.LDAPConfig.Enabled() {
+		perms, _ := processLDAPAuthentication(key, pass, user)
+		if perms != nil {
+			return perms, nil
+		}
+	}
+
+internalAuth:
+	ui, ok := globalIAMSys.GetUser(context.Background(), user)
+	if !ok {
+		return nil, errNoSuchUser
+	}
+
+	if caPublicKey != nil {
+		err := validateKey(c, key)
+		if err != nil {
+			return nil, errAuthentication
+		}
+	} else {
+
+		// Temporary credentials are not allowed.
+		if ui.Credentials.IsTemp() {
+			return nil, errAuthentication
+		}
+
+		if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) != 1 {
+			return nil, errAuthentication
+		}
+	}
+
+	return &ssh.Permissions{
+		CriticalOptions: map[string]string{
+			"AccessKey":    ui.Credentials.AccessKey,
+			"SecretKey":    ui.Credentials.SecretKey,
+			"SessionToken": ui.Credentials.SessionToken,
+		},
+		Extensions: make(map[string]string),
+	}, nil
+}
+
+func processLDAPAuthentication(key ssh.PublicKey, pass []byte, user string) (perms *ssh.Permissions, err error) {
+	var lookupResult *xldap.DNSearchResult
+	var targetGroups []string
+
+	if pass == nil && key == nil {
+		return nil, errAuthentication
+	}
+
+	if pass != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), user)
+		if err == nil {
+			if subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), pass) != 1 {
+				return nil, errAuthentication
+			}
+
+			return &ssh.Permissions{
+				CriticalOptions: map[string]string{
+					"AccessKey":    sa.Credentials.AccessKey,
+					"SecretKey":    sa.Credentials.SecretKey,
+					"SessionToken": sa.Credentials.SessionToken,
+				},
+				Extensions: make(map[string]string),
+			}, nil
+		}
+
+		if !errors.Is(err, errNoSuchServiceAccount) {
+			return nil, err
+		}
+
+		lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.Bind(user, string(pass))
+		if err != nil {
+			return nil, err
+		}
+
+	} else if key != nil {
+
+		lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(user)
+		if err != nil {
+			return nil, err
+		}
+
+	}
+
+	if lookupResult == nil {
+		return nil, errNoSuchUser
+	}
+
+	ldapPolicies, _ := globalIAMSys.PolicyDBGet(lookupResult.NormDN, targetGroups...)
+	if len(ldapPolicies) == 0 {
+		return nil, errSFTPUserHasNoPolicies
+	}
+
+	claims := make(map[string]interface{})
+	for attribKey, attribValue := range lookupResult.Attributes {
+		// we skip multi-value attributes here, as they cannot
+		// be stored in the critical options.
+		if len(attribValue) != 1 {
+			continue
+		}
+
+		if attribKey == "sshPublicKey" && key != nil {
+			key2, _, _, _, err := ssh.ParseAuthorizedKey([]byte(attribValue[0]))
+			if err != nil {
+				return nil, errSFTPPublicKeyBadFormat
+			}
+
+			if subtle.ConstantTimeCompare(key2.Marshal(), key.Marshal()) != 1 {
+				return nil, errAuthentication
+			}
+		}
+		claims[ldapAttribPrefix+attribKey] = attribValue[0]
+	}
+
+	expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
+	if err != nil {
+		return nil, err
+	}
+
+	claims[expClaim] = UTCNow().Add(expiryDur).Unix()
+	claims[ldapUserN] = user
+	claims[ldapUser] = lookupResult.NormDN
+
+	cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+
+	// Set the parent of the temporary access key, this is useful
+	// in obtaining service accounts by this cred.
+	cred.ParentUser = lookupResult.NormDN
+
+	// Set this value to LDAP groups, LDAP user can be part
+	// of large number of groups
+	cred.Groups = targetGroups
+
+	// Set the newly generated credentials, policyName is empty on purpose
+	// LDAP policies are applied automatically using their ldapUser, ldapGroups
+	// mapping.
+	updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
+	if err != nil {
+		return nil, err
+	}
+
+	replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+		Type: madmin.SRIAMItemSTSAcc,
+		STSCredential: &madmin.SRSTSCredential{
+			AccessKey:    cred.AccessKey,
+			SecretKey:    cred.SecretKey,
+			SessionToken: cred.SessionToken,
+			ParentUser:   cred.ParentUser,
+		},
+		UpdatedAt: updatedAt,
+	}))
+
+	return &ssh.Permissions{
+		CriticalOptions: map[string]string{
+			"AccessKey":    cred.AccessKey,
+			"SecretKey":    cred.SecretKey,
+			"SessionToken": cred.SessionToken,
+		},
+		Extensions: make(map[string]string),
+	}, nil
+}
+
+func validateKey(c ssh.ConnMetadata, clientKey ssh.PublicKey) (err error) {
+	if caPublicKey == nil {
+		return errors.New("public key authority validation requested but no ca public key specified.")
+	}
+
+	cert, ok := clientKey.(*ssh.Certificate)
+	if !ok {
+		return errSftpPublicKeyWithoutCert
+	}
+
+	// ssh.CheckCert called by ssh.Authenticate accepts certificates
+	// with empty principles list so we block those in here.
+	if len(cert.ValidPrincipals) == 0 {
+		return errSftpCertWithoutPrincipals
+	}
+
+	// Verify that certificate provided by user is issued by trusted CA,
+	// username in authentication request matches to identities in certificate
+	// and that certificate type is correct.
+	checker := ssh.CertChecker{}
+	checker.IsUserAuthority = func(k ssh.PublicKey) bool {
+		return subtle.ConstantTimeCompare(caPublicKey.Marshal(), k.Marshal()) == 1
+	}
+
+	_, err = checker.Authenticate(c, clientKey)
+	return
+}
+
+type sftpLogger struct{}
+
+func (s *sftpLogger) Info(tag xsftp.LogType, msg string) {
+	logger.Info(msg)
+}
+
 func (s *sftpLogger) Error(tag xsftp.LogType, err error) {
 	switch tag {
 	case xsftp.AcceptNetworkError:
@@ -160,16 +386,19 @@ func filterAlgos(arg string, want []string, allowed []string) []string {
 
 func startSFTPServer(args []string) {
 	var (
-		port          int
-		publicIP      string
-		sshPrivateKey string
-		userCaKeyFile string
+		port            int
+		publicIP        string
+		sshPrivateKey   string
+		userCaKeyFile   string
+		disablePassAuth bool
 	)
+
 	allowPubKeys := supportedPubKeyAuthAlgos
 	allowKexAlgos := preferredKexAlgos
 	allowCiphers := preferredCiphers
 	allowMACs := supportedMACs
 	var err error
+
 	for _, arg := range args {
 		tokens := strings.SplitN(arg, "=", 2)
 		if len(tokens) != 2 {
@@ -201,6 +430,8 @@ func startSFTPServer(args []string) {
 			allowMACs = filterAlgos(arg, strings.Split(tokens[1], ","), supportedMACs)
 		case "trusted-user-ca-key":
 			userCaKeyFile = tokens[1]
+		case "password-auth":
+			disablePassAuth, _ = strconv.ParseBool(tokens[1])
 		}
 	}
 
@@ -222,6 +453,18 @@ func startSFTPServer(args []string) {
 		logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not parseable: %v", err), "unable to start SFTP server")
 	}
 
+	if userCaKeyFile != "" {
+		keyBytes, err := os.ReadFile(userCaKeyFile)
+		if err != nil {
+			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not accessible: %v", err), "unable to start SFTP server")
+		}
+
+		caPublicKey, _, _, _, err = ssh.ParseAuthorizedKey(keyBytes)
+		if err != nil {
+			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not parseable: %v", err), "unable to start SFTP server")
+		}
+	}
+
 	// An SSH server is represented by a ServerConfig, which holds
 	// certificate details and handles authentication of ServerConns.
 	sshConfig := &ssh.ServerConfig{
@@ -231,116 +474,13 @@ func startSFTPServer(args []string) {
 			MACs:         allowMACs,
 		},
 		PublicKeyAuthAlgorithms: allowPubKeys,
-		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
-			if globalIAMSys.LDAPConfig.Enabled() {
-				sa, _, err := globalIAMSys.getServiceAccount(context.Background(), c.User())
-				if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
-					return nil, err
-				}
-				if errors.Is(err, errNoSuchServiceAccount) {
-					lookupResult, targetGroups, err := globalIAMSys.LDAPConfig.Bind(c.User(), string(pass))
-					if err != nil {
-						return nil, err
-					}
-					targetUser := lookupResult.NormDN
-					ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
-					if len(ldapPolicies) == 0 {
-						return nil, errAuthentication
-					}
-					criticalOptions := map[string]string{
-						ldapUser:       targetUser,
-						ldapActualUser: lookupResult.ActualDN,
-						ldapUserN:      c.User(),
-					}
-					for attribKey, attribValue := range lookupResult.Attributes {
-						// we skip multi-value attributes here, as they cannot
-						// be stored in the critical options.
-						if len(attribValue) == 1 {
-							criticalOptions[ldapAttribPrefix+attribKey] = attribValue[0]
-						}
-					}
-
-					return &ssh.Permissions{
-						CriticalOptions: criticalOptions,
-						Extensions:      make(map[string]string),
-					}, nil
-				}
-				if subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), pass) == 1 {
-					return &ssh.Permissions{
-						CriticalOptions: map[string]string{
-							"accessKey": c.User(),
-						},
-						Extensions: make(map[string]string),
-					}, nil
-				}
-				return nil, errAuthentication
-			}
-
-			ui, ok := globalIAMSys.GetUser(context.Background(), c.User())
-			if !ok {
-				return nil, errNoSuchUser
-			}
-
-			if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) == 1 {
-				return &ssh.Permissions{
-					CriticalOptions: map[string]string{
-						"accessKey": c.User(),
-					},
-					Extensions: make(map[string]string),
-				}, nil
-			}
-			return nil, errAuthentication
-		},
+		PublicKeyCallback:       sshPubKeyAuth,
 	}
 
-	if userCaKeyFile != "" {
-		keyBytes, err := os.ReadFile(userCaKeyFile)
-		if err != nil {
-			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not accessible: %v", err), "unable to start SFTP server")
-		}
-
-		caPublicKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes)
-		if err != nil {
-			logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not parseable: %v", err), "unable to start SFTP server")
-		}
-
-		sshConfig.PublicKeyCallback = func(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
-			_, ok := globalIAMSys.GetUser(context.Background(), c.User())
-			if !ok {
-				return nil, errNoSuchUser
-			}
-
-			// Verify that client provided certificate, not only public key.
-			cert, ok := key.(*ssh.Certificate)
-			if !ok {
-				return nil, errSftpPublicKeyWithoutCert
-			}
-
-			// ssh.CheckCert called by ssh.Authenticate accepts certificates
-			// with empty principles list so we block those in here.
-			if len(cert.ValidPrincipals) == 0 {
-				return nil, errSftpCertWithoutPrincipals
-			}
-
-			// Verify that certificate provided by user is issued by trusted CA,
-			// username in authentication request matches to identities in certificate
-			// and that certificate type is correct.
-			checker := ssh.CertChecker{}
-			checker.IsUserAuthority = func(k ssh.PublicKey) bool {
-				return bytes.Equal(k.Marshal(), caPublicKey.Marshal())
-			}
-			_, err = checker.Authenticate(c, key)
-			if err != nil {
-				return nil, err
-			}
-
-			return &ssh.Permissions{
-				CriticalOptions: map[string]string{
-					"accessKey": c.User(),
-				},
-				Extensions: make(map[string]string),
-			}, nil
-		}
+	if !disablePassAuth {
+		sshConfig.PasswordCallback = sshPasswordAuth
+	} else {
+		sshConfig.PasswordCallback = nil
 	}
 
 	sshConfig.AddHostKey(private)

cmd/sftp-server_test.go

@@ -0,0 +1,289 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/minio/madmin-go/v3"
+	"golang.org/x/crypto/ssh"
+)
+
+type MockConnMeta struct {
+	username string
+}
+
+func (m *MockConnMeta) User() string {
+	return m.username
+}
+
+func (m *MockConnMeta) SessionID() []byte {
+	return []byte{}
+}
+
+func (m *MockConnMeta) ClientVersion() []byte {
+	return []byte{}
+}
+
+func (m *MockConnMeta) ServerVersion() []byte {
+	return []byte{}
+}
+
+func (m *MockConnMeta) RemoteAddr() net.Addr {
+	return nil
+}
+
+func (m *MockConnMeta) LocalAddr() net.Addr {
+	return nil
+}
+
+func newSSHConnMock(username string) ssh.ConnMetadata {
+	return &MockConnMeta{username: username}
+}
+
+func TestSFTPAuthentication(t *testing.T) {
+	for i, testCase := range iamTestSuites {
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
+			func(t *testing.T) {
+				c := &check{t, testCase.serverType}
+				suite := testCase
+
+				suite.SetUpSuite(c)
+
+				suite.SFTPServiceAccountLogin(c)
+				suite.SFTPInvalidServiceAccountPassword(c)
+
+				// LDAP tests
+				ldapServer := os.Getenv(EnvTestLDAPServer)
+				if ldapServer == "" {
+					c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer)
+				}
+
+				suite.SetUpLDAP(c, ldapServer)
+
+				suite.SFTPFailedAuthDueToMissingPolicy(c)
+				suite.SFTPFailedAuthDueToInvalidUser(c)
+				suite.SFTPFailedForcedServiceAccountAuthOnLDAPUser(c)
+				suite.SFTPFailedAuthDueToInvalidPassword(c)
+
+				suite.SFTPValidLDAPLoginWithPassword(c)
+
+				suite.SFTPPublicKeyAuthentication(c)
+				suite.SFTPFailedPublicKeyAuthenticationInvalidKey(c)
+
+				suite.TearDownSuite(c)
+			},
+		)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedPublicKeyAuthenticationInvalidKey(c *check) {
+	keyBytes, err := os.ReadFile("./testdata/invalid_test_key.pub")
+	if err != nil {
+		c.Fatalf("could not read test key file: %s", err)
+	}
+
+	testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes)
+	if err != nil {
+		c.Fatalf("could not parse test key file: %s", err)
+	}
+
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err == nil || !errors.Is(err, errAuthentication) {
+		c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPPublicKeyAuthentication(c *check) {
+	keyBytes, err := os.ReadFile("./testdata/dillon_test_key.pub")
+	if err != nil {
+		c.Fatalf("could not read test key file: %s", err)
+	}
+
+	testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes)
+	if err != nil {
+		c.Fatalf("could not parse test key file: %s", err)
+	}
+
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err != nil {
+		c.Fatalf("expected no error but got(%s)", err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPubKeyAuth(newSSHCon, testKey)
+	if err != nil {
+		c.Fatalf("expected no error but got(%s)", err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedAuthDueToMissingPolicy(c *check) {
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err == nil || !errors.Is(err, errSFTPUserHasNoPolicies) {
+		c.Fatalf("expected err(%s) but got (%s)", errSFTPUserHasNoPolicies, err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedAuthDueToInvalidUser(c *check) {
+	newSSHCon := newSSHConnMock("dillon_error")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon_error"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedForcedServiceAccountAuthOnLDAPUser(c *check) {
+	newSSHCon := newSSHConnMock("dillon=svc")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPFailedAuthDueToInvalidPassword(c *check) {
+	newSSHCon := newSSHConnMock("dillon")
+	_, err := sshPasswordAuth(newSSHCon, []byte("dillon_error"))
+	if err == nil || !errors.Is(err, errNoSuchUser) {
+		c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPInvalidServiceAccountPassword(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	if err != nil {
+		c.Fatalf("unable to set policy: %v", err)
+	}
+
+	newSSHCon := newSSHConnMock(accessKey + "=svc")
+	_, err = sshPasswordAuth(newSSHCon, []byte("invalid"))
+	if err == nil || !errors.Is(err, errAuthentication) {
+		c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err)
+	}
+
+	newSSHCon = newSSHConnMock(accessKey)
+	_, err = sshPasswordAuth(newSSHCon, []byte("invalid"))
+	if err == nil || !errors.Is(err, errAuthentication) {
+		c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPServiceAccountLogin(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	if err != nil {
+		c.Fatalf("unable to set policy: %v", err)
+	}
+
+	newSSHCon := newSSHConnMock(accessKey + "=svc")
+	_, err = sshPasswordAuth(newSSHCon, []byte(secretKey))
+	if err != nil {
+		c.Fatalf("expected no error but got (%s)", err)
+	}
+
+	newSSHCon = newSSHConnMock(accessKey)
+	_, err = sshPasswordAuth(newSSHCon, []byte(secretKey))
+	if err != nil {
+		c.Fatalf("expected no error but got (%s)", err)
+	}
+}
+
+func (s *TestSuiteIAM) SFTPValidLDAPLoginWithPassword(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	// we need to do this so that the user has a policy before authentication.
+	// ldap user accounts without policies are denied access in sftp.
+	policy := "mypolicy"
+	policyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::BUCKET/*"
+   ]
+  }
+ ]
+}`)
+
+	err := s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
+	err = s.adm.SetPolicy(ctx, policy, userDN, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	newSSHCon := newSSHConnMock("dillon=ldap")
+	_, err = sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err != nil {
+		c.Fatal("Password authentication failed for user (dillon):", err)
+	}
+
+	newSSHCon = newSSHConnMock("dillon")
+	_, err = sshPasswordAuth(newSSHCon, []byte("dillon"))
+	if err != nil {
+		c.Fatal("Password authentication failed for user (dillon):", err)
+	}
+}

cmd/sts-handlers.go

@@ -22,9 +22,11 @@ import (
 	"context"
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -82,8 +84,50 @@ const (
 
 	// Role Claim key
 	roleArnClaim = "roleArn"
+
+	// maximum supported STS session policy size
+	maxSTSSessionPolicySize = 2048
 )
 
+type stsClaims map[string]interface{}
+
+func (c stsClaims) populateSessionPolicy(form url.Values) error {
+	if len(form) == 0 {
+		return nil
+	}
+
+	sessionPolicyStr := form.Get(stsPolicy)
+	if len(sessionPolicyStr) == 0 {
+		return nil
+	}
+
+	sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
+	if err != nil {
+		return err
+	}
+
+	// Version in policy must not be empty
+	if sessionPolicy.Version == "" {
+		return errors.New("Version cannot be empty expecting '2012-10-17'")
+	}
+
+	policyBuf, err := json.Marshal(sessionPolicy)
+	if err != nil {
+		return err
+	}
+
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+	// The plain text that you use for both inline and managed session
+	// policies shouldn't exceed maxSTSSessionPolicySize characters.
+	if len(policyBuf) > maxSTSSessionPolicySize {
+		return errSessionPolicyTooLarge
+	}
+
+	c[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
+	return nil
+}
+
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
 type stsAPIHandlers struct{}
 
@@ -212,7 +256,7 @@ func getTokenSigningKey() (string, error) {
 func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRole")
 
-	claims := make(map[string]interface{})
+	claims := stsClaims{}
 	defer logger.AuditLog(ctx, w, r, claims)
 
 	// Check auth here (otherwise r.Form will have unexpected values from
@@ -249,29 +293,11 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	sessionPolicyStr := r.Form.Get(stsPolicy)
-	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
-	// The plain text that you use for both inline and managed session
-	// policies shouldn't exceed 2048 characters.
-	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, errSessionPolicyTooLarge)
+	if err := claims.populateSessionPolicy(r.Form); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
-		if err != nil {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
-			return
-		}
-
-		// Version in policy must not be empty
-		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Version cannot be empty expecting '2012-10-17'"))
-			return
-		}
-	}
-
 	duration, err := openid.GetDefaultExpiration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
@@ -288,10 +314,6 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		claims[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	secret, err := getTokenSigningKey()
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
@@ -342,7 +364,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRoleSSOCommon")
 
-	claims := make(map[string]interface{})
+	claims := stsClaims{}
 	defer logger.AuditLog(ctx, w, r, claims)
 
 	// Parse the incoming form data.
@@ -449,31 +471,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		claims[iamPolicyClaimNameOpenID()] = policyName
 	}
 
-	sessionPolicyStr := r.Form.Get(stsPolicy)
-	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
-	// The plain text that you use for both inline and managed session
-	// policies shouldn't exceed 2048 characters.
-	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
+	if err := claims.populateSessionPolicy(r.Form); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
-		if err != nil {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
-			return
-		}
-
-		// Version in policy must not be empty
-		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid session policy version"))
-			return
-		}
-
-		claims[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	secret, err := getTokenSigningKey()
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
@@ -612,7 +614,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *
 func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRoleWithLDAPIdentity")
 
-	claims := make(map[string]interface{})
+	claims := stsClaims{}
 	defer logger.AuditLog(ctx, w, r, claims, stsLDAPPassword)
 
 	// Parse the incoming form data.
@@ -643,29 +645,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		return
 	}
 
-	sessionPolicyStr := r.Form.Get(stsPolicy)
-	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
-	// The plain text that you use for both inline and managed session
-	// policies shouldn't exceed 2048 characters.
-	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
+	if err := claims.populateSessionPolicy(r.Form); err != nil {
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
-		if err != nil {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
-			return
-		}
-
-		// Version in policy must not be empty
-		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Version needs to be specified in session policy"))
-			return
-		}
-	}
-
 	if !globalIAMSys.Initialized() {
 		writeSTSErrorResponse(ctx, w, ErrSTSIAMNotInitialized, errIAMNotInitialized)
 		return
@@ -708,10 +692,6 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		claims[ldapAttribPrefix+attrib] = value
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		claims[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	secret, err := getTokenSigningKey()
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)

cmd/test-utils_test.go

@@ -106,7 +106,7 @@ func TestMain(m *testing.M) {
 	// logger.AddTarget(console.New())
 
 	// Set system resources to maximum.
-	setMaxResources(nil)
+	setMaxResources(serverCtxt{})
 
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background(), io.Discard)

cmd/testdata/dillon_test_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVGk/SRz4fwTPK0+Ra7WYUGf3o08YkpI0yTMPpHwYoq dillon@example.io

cmd/testdata/invalid_test_key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDES4saDDRpoHDVmiYESEQrCYhw8EK7Utj/A/lqxiqZlP6Il3aN2fWu6uJQdWAovZxNeXUf8LIujisW1mJWGZPql0SLKVq6IZ707OAGmKA59IXfF5onRoU9+K4UDL7BJFfix6/3F5OV2WB3ChFrOrXhJ0CZ0sVAfGcV4q72kS19YjZNX3fqCc2HF8UQEaZGKIkw5MtdZI9a1P2bqnPuPGJybRFUzyoQXPge45QT5jnpcsAXOuXcGxbjuqaaHXFNTSKAkCU93TcjAbqUMkTz2mnFz/MnrKJTECN3Fy0GPCCQ5dxmG8p8DyMiNl7JYkX2r3XYgxmioCzkcg8fDs5p0CaQcipu+MA7iK7APKq7v4Zr/wNltXHI3DE9S8J88Hxb2FZAyEhCRfcgGmCVfoZxVNCRHNkGYzfe63BkxtnseUCzpYEhKv02H5u9rjFpdMY37kDfHDVqBbgutdMij+tQAEp1kyqi6TQL+4XHjPHkLaeekW07yB+VI90dK1A9dzTpOvE= liza@example.io

cmd/utils.go

@@ -38,7 +38,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/go-oidc"
+	"github.com/coreos/go-oidc/v3/oidc"
 	"github.com/dustin/go-humanize"
 	"github.com/felixge/fgprof"
 	"github.com/minio/madmin-go/v3"

cmd/xl-storage-disk-id-check.go

@@ -288,7 +288,7 @@ func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context, opts DiskInfoOption
 	}
 
 	si := p.updateStorageMetrics(storageMetricDiskInfo)
-	defer si(&err)
+	defer si(0, &err)
 
 	if opts.NoOp {
 		if opts.Metrics {
@@ -341,7 +341,7 @@ func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...strin
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.MakeVolBulk(ctx, volumes...) })
@@ -352,7 +352,7 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.MakeVol(ctx, volume) })
@@ -363,7 +363,7 @@ func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) (vi []VolInfo, err
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.ListVols(ctx)
 }
@@ -373,7 +373,7 @@ func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol
 	if err != nil {
 		return vol, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return xioutil.WithDeadline[VolInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result VolInfo, err error) {
 		return p.storage.StatVol(ctx, volume)
@@ -385,7 +385,7 @@ func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, for
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.DeleteVol(ctx, volume, forceDelete) })
@@ -396,7 +396,7 @@ func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, origvolume, volume,
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.ListDir(ctx, origvolume, volume, dirPath, count)
 }
@@ -407,7 +407,9 @@ func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path
 	if err != nil {
 		return 0, err
 	}
-	defer done(&err)
+	defer func() {
+		done(n, &err)
+	}()
 
 	return xioutil.WithDeadline[int64](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result int64, err error) {
 		return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
@@ -420,7 +422,7 @@ func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, pa
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(int64(len(buf)), &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error {
@@ -433,7 +435,7 @@ func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, origvolume, volum
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(size, &err)
 
 	return p.storage.CreateFile(ctx, origvolume, volume, path, size, io.NopCloser(reader))
 }
@@ -443,7 +445,7 @@ func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(length, &err)
 
 	return xioutil.WithDeadline[io.ReadCloser](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result io.ReadCloser, err error) {
 		return p.storage.ReadFileStream(ctx, volume, path, offset, length)
@@ -455,7 +457,7 @@ func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPat
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) })
@@ -470,7 +472,7 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
 		if err == nil && !skipAccessChecks(dstVolume) {
 			p.storage.setWriteAttribute(p.totalWrites.Add(1))
 		}
-		done(&err)
+		done(0, &err)
 	}()
 
 	// Copy inline data to a new buffer to function with deadlines.
@@ -490,7 +492,7 @@ func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, pa
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.CheckParts(ctx, volume, path, fi) })
@@ -501,7 +503,7 @@ func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path s
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.Delete(ctx, volume, path, deleteOpts) })
@@ -548,7 +550,7 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 				p.storage.setDeleteAttribute(p.totalDeletes.Add(permanentDeletes))
 			}
 		}
-		done(&err)
+		done(0, &err)
 	}()
 
 	errs = p.storage.DeleteVersions(ctx, volume, versions, opts)
@@ -567,7 +569,7 @@ func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path stri
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.VerifyFile(ctx, volume, path, fi)
 }
@@ -577,7 +579,7 @@ func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(int64(len(b)), &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.WriteAll(ctx, volume, path, b) })
@@ -589,7 +591,7 @@ func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path s
 		return err
 	}
 	defer func() {
-		defer done(&err)
+		defer done(0, &err)
 
 		if err == nil && !skipAccessChecks(volume) {
 			if opts.UndoWrite {
@@ -616,7 +618,7 @@ func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.UpdateMetadata(ctx, volume, path, fi, opts) })
@@ -627,7 +629,7 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, vo
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.WriteMetadata(ctx, origvolume, volume, path, fi) })
@@ -638,7 +640,7 @@ func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, origvolume, volu
 	if err != nil {
 		return fi, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return xioutil.WithDeadline[FileInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result FileInfo, err error) {
 		return p.storage.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
@@ -650,7 +652,11 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	var sz int
+	defer func() {
+		sz = len(buf)
+		done(int64(sz), &err)
+	}()
 
 	return xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result []byte, err error) {
 		return p.storage.ReadAll(ctx, volume, path)
@@ -662,7 +668,9 @@ func (p *xlStorageDiskIDCheck) ReadXL(ctx context.Context, volume string, path s
 	if err != nil {
 		return RawFileInfo{}, err
 	}
-	defer done(&err)
+	defer func() {
+		done(int64(len(rf.Buf)), &err)
+	}()
 
 	return xioutil.WithDeadline[RawFileInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result RawFileInfo, err error) {
 		return p.storage.ReadXL(ctx, volume, path, readData)
@@ -674,7 +682,7 @@ func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path st
 	if err != nil {
 		return nil, err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.StatInfoFile(ctx, volume, path, glob)
 }
@@ -689,7 +697,7 @@ func (p *xlStorageDiskIDCheck) ReadMultiple(ctx context.Context, req ReadMultipl
 		xioutil.SafeClose(resp)
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	return p.storage.ReadMultiple(ctx, req, resp)
 }
@@ -701,19 +709,20 @@ func (p *xlStorageDiskIDCheck) CleanAbandonedData(ctx context.Context, volume st
 	if err != nil {
 		return err
 	}
-	defer done(&err)
+	defer done(0, &err)
 
 	w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 	return w.Run(func() error { return p.storage.CleanAbandonedData(ctx, volume, path) })
 }
 
-func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string, err string, custom map[string]string) madmin.TraceInfo {
+func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string, size int64, err string, custom map[string]string) madmin.TraceInfo {
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceStorage,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  "storage." + s.String(),
 		Duration:  duration,
+		Bytes:     size,
 		Path:      path,
 		Error:     err,
 		Custom:    custom,
@@ -733,10 +742,10 @@ func scannerTrace(s scannerMetric, startTime time.Time, duration time.Duration,
 }
 
 // Update storage metrics
-func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(err *error) {
+func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(sz int64, err *error) {
 	startTime := time.Now()
 	trace := globalTrace.NumSubscribers(madmin.TraceStorage) > 0
-	return func(errp *error) {
+	return func(sz int64, errp *error) {
 		duration := time.Since(startTime)
 
 		var err error
@@ -767,7 +776,7 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 			}
 			custom["total-errs-timeout"] = strconv.FormatUint(p.totalErrsTimeout.Load(), 10)
 			custom["total-errs-availability"] = strconv.FormatUint(p.totalErrsAvailability.Load(), 10)
-			globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " "), errStr, custom))
+			globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " "), sz, errStr, custom))
 		}
 	}
 }
@@ -824,7 +833,7 @@ func (h *healthDiskCtxValue) logSuccess() {
 
 // noopDoneFunc is a no-op done func.
 // Can be reused.
-var noopDoneFunc = func(_ *error) {}
+var noopDoneFunc = func(_ int64, _ *error) {}
 
 // TrackDiskHealth for this request.
 // When a non-nil error is returned 'done' MUST be called
@@ -833,7 +842,7 @@ var noopDoneFunc = func(_ *error) {}
 // is either nil or io.EOF the disk is considered good.
 // So if unsure if the disk status is ok, return nil as a parameter to done.
 // Shadowing will work as long as return error is named: https://go.dev/play/p/sauq86SsTN2
-func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(*error), err error) {
+func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(int64, *error), err error) {
 	done = noopDoneFunc
 	if contextCanceled(ctx) {
 		return ctx, done, ctx.Err()
@@ -866,7 +875,7 @@ func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMet
 	ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess})
 	si := p.updateStorageMetrics(s, paths...)
 	var once sync.Once
-	return ctx, func(errp *error) {
+	return ctx, func(sz int64, errp *error) {
 		p.health.waiting.Add(-1)
 		once.Do(func() {
 			if errp != nil {
@@ -875,7 +884,7 @@ func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMet
 					p.health.logSuccess()
 				}
 			}
-			si(errp)
+			si(sz, errp)
 		})
 	}, nil
 }

cmd/xl-storage-format-v2.go

@@ -250,7 +250,7 @@ type xlMetaV2VersionHeader struct {
 	Signature [4]byte
 	Type      VersionType
 	Flags     xlFlags
-	EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
+	EcN, EcM  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 }
 
 func (x xlMetaV2VersionHeader) String() string {
@@ -368,8 +368,8 @@ func (j *xlMetaV2Version) header() xlMetaV2VersionHeader {
 		Signature: j.getSignature(),
 		Type:      j.Type,
 		Flags:     flags,
-		EcN:       ecM,
-		EcM:       ecN,
+		EcN:       ecN,
+		EcM:       ecM,
 	}
 }
 

cmd/xl-storage-format-v2_gen.go

@@ -2235,16 +2235,16 @@ func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		z.Flags = xlFlags(zb0003)
 	}
-	z.EcM, err = dc.ReadUint8()
-	if err != nil {
-		err = msgp.WrapError(err, "EcM")
-		return
-	}
 	z.EcN, err = dc.ReadUint8()
 	if err != nil {
 		err = msgp.WrapError(err, "EcN")
 		return
 	}
+	z.EcM, err = dc.ReadUint8()
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
 	return
 }
 
@@ -2280,16 +2280,16 @@ func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Flags")
 		return
 	}
-	err = en.WriteUint8(z.EcM)
-	if err != nil {
-		err = msgp.WrapError(err, "EcM")
-		return
-	}
 	err = en.WriteUint8(z.EcN)
 	if err != nil {
 		err = msgp.WrapError(err, "EcN")
 		return
 	}
+	err = en.WriteUint8(z.EcM)
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
 	return
 }
 
@@ -2303,8 +2303,8 @@ func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendBytes(o, (z.Signature)[:])
 	o = msgp.AppendUint8(o, uint8(z.Type))
 	o = msgp.AppendUint8(o, uint8(z.Flags))
-	o = msgp.AppendUint8(o, z.EcM)
 	o = msgp.AppendUint8(o, z.EcN)
+	o = msgp.AppendUint8(o, z.EcM)
 	return
 }
 
@@ -2353,16 +2353,16 @@ func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		z.Flags = xlFlags(zb0003)
 	}
-	z.EcM, bts, err = msgp.ReadUint8Bytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err, "EcM")
-		return
-	}
 	z.EcN, bts, err = msgp.ReadUint8Bytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err, "EcN")
 		return
 	}
+	z.EcM, bts, err = msgp.ReadUint8Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "EcM")
+		return
+	}
 	o = bts
 	return
 }

docs/bucket/replication/test_del_marker_proxying.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+	cleanup
+
+	for site in sitea siteb; do
+		echo "$site server logs ========="
+		cat "/tmp/${site}_1.log"
+		echo "==========================="
+		cat "/tmp/${site}_2.log"
+	done
+
+	exit 1
+}
+
+cleanup() {
+	echo -n "Cleaning up instances of MinIO ..."
+	pkill -9 minio || sudo pkill -9 minio
+	rm -rf /tmp/sitea
+	rm -rf /tmp/siteb
+	echo "done"
+}
+
+cleanup
+
+export MINIO_CI_CD=1
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+
+# Start MinIO instances
+echo -n "Starting MinIO instances ..."
+minio server --address ":9001" --console-address ":10000" /tmp/sitea/{1...4}/disk{1...4} /tmp/sitea/{5...8}/disk{1...4} >/tmp/sitea_1.log 2>&1 &
+minio server --address ":9002" --console-address ":11000" /tmp/siteb/{1...4}/disk{1...4} /tmp/siteb/{5...8}/disk{1...4} >/tmp/siteb_1.log 2>&1 &
+echo "done"
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001
+export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9002
+
+./mc ready sitea
+./mc ready siteb
+
+./mc mb sitea/bucket
+./mc version enable sitea/bucket
+./mc mb siteb/bucket
+./mc version enable siteb/bucket
+
+# Set bucket replication
+./mc replicate add sitea/bucket --remote-bucket siteb/bucket
+
+# Run the test to make sure proxying of DEL marker doesn't happen
+loop_count=0
+while true; do
+	if [ $loop_count -eq 100 ]; then
+		break
+	fi
+	echo "Hello World" | ./mc pipe sitea/bucket/obj$loop_count
+	./mc rm sitea/bucket/obj$loop_count
+	RESULT=$({ ./mc stat sitea/bucket/obj$loop_count; } 2>&1)
+	if [[ ${RESULT} != *"Object does not exist"* ]]; then
+		echo "BUG: stat should fail. succeeded."
+		exit_1
+	fi
+	loop_count=$((loop_count + 1))
+done
+
+cleanup

docs/debugging/inspect/main.go

@@ -59,7 +59,7 @@ func main() {
 		}
 
 		// Prompt for decryption key if no --key or --private-key are provided
-		if len(privateKey) == 0 {
+		if len(privateKey) == 0 && !*stdin {
 			reader := bufio.NewReader(os.Stdin)
 			fmt.Print("Enter Decryption Key: ")
 

docs/debugging/xl-meta/main.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/md5"
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
@@ -710,7 +711,7 @@ type xlMetaV2VersionHeaderV2 struct {
 	Signature [4]byte
 	Type      uint8
 	Flags     uint8
-	EcM, EcN  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
+	EcN, EcM  uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
@@ -768,19 +769,19 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt
 			var zb0004 uint8
 			zb0004, bts, err = msgp.ReadUint8Bytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "EcM")
+				err = msgp.WrapError(err, "EcN")
 				return
 			}
-			z.EcM = zb0004
+			z.EcN = zb0004
 		}
 		{
 			var zb0005 uint8
 			zb0005, bts, err = msgp.ReadUint8Bytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "EcN")
+				err = msgp.WrapError(err, "EcM")
 				return
 			}
-			z.EcN = zb0005
+			z.EcM = zb0005
 		}
 	}
 	o = bts
@@ -814,6 +815,7 @@ type mappedData struct {
 	blockOffset                int // Offset in bytes to start of block.
 	blocks                     int // 0 = one block.
 	objSize, partSize          int
+	wantMD5                    string
 }
 
 func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
@@ -835,6 +837,9 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 				EcBSize   int
 				PartNums  []int
 				PartSizes []int
+				MetaUsr   struct {
+					Etag string `json:"etag"`
+				}
 			}
 		}
 		var ei erasureInfo
@@ -845,11 +850,14 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 			}
 			m.data = ei.V2Obj.EcM
 			m.parity = ei.V2Obj.EcN
+			if len(ei.V2Obj.PartNums) == 1 && !strings.ContainsRune(ei.V2Obj.MetaUsr.Etag, '-') {
+				m.wantMD5 = ei.V2Obj.MetaUsr.Etag
+			}
 			if m.shards == 0 {
 				m.shards = m.data + m.parity
 			}
 			idx = ei.V2Obj.EcIndex - 1
-			fmt.Println("Read shard", ei.V2Obj.EcIndex, "Data shards", m.data, "Parity", m.parity, fmt.Sprintf("(%s)", file))
+			fmt.Println("Read shard", ei.V2Obj.EcIndex, fmt.Sprintf("(%s)", file))
 			if ei.V2Obj.Size != m.objSize {
 				return nil, fmt.Errorf("size mismatch. Meta size: %d, Prev: %d", ei.V2Obj.Size, m.objSize)
 			}
@@ -893,7 +901,6 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 		if err != nil {
 			return nil, err
 		}
-		fmt.Println("Block data size:", m.size, "Shard size", ssz, "Got Shard:", len(b), "Bitrot ok")
 
 		if m.mapped == nil {
 			m.mapped = make([]byte, m.size)
@@ -912,6 +919,7 @@ func readAndMap(files []string, partNum, blockNum int) (*mappedData, error) {
 		if start >= len(m.mapped) {
 			continue
 		}
+		fmt.Println("Block data size:", m.size, "Shard size", ssz, "Got Shard:", len(b), "Bitrot ok", "Start", start, "End", start+len(b))
 		copy(m.mapped[start:], b)
 		for j := range b {
 			if j+start >= len(m.filled) {
@@ -1038,14 +1046,23 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 		if len(files[partIdx]) == 0 {
 			continue
 		}
+		var wantMD5 string
 		exportedSizes := make(map[int]bool)
+		// block -> data
+		combineSharedBlocks := make(map[int][]byte)
+		combineFilledBlocks := make(map[int][]byte)
 	nextFile:
 		for key, file := range files[partIdx] {
 			fmt.Println("Reading base version", file[0], "part", part)
 			var combined []byte
 			var missingAll int
 			var lastValidAll int
+
+			attempt := 0
 			for block := 0; ; block++ {
+				combineFilled := combineFilledBlocks[block]
+				combineShared := combineSharedBlocks[block]
+			nextAttempt:
 				fmt.Printf("Block %d, Base version %q. Part %d. Files %d\n", block+1, key, part, len(file))
 				m, err := readAndMap(file, part, block)
 				if err != nil {
@@ -1055,11 +1072,30 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 					fmt.Println("Skipping version", key, "as it has already been exported.")
 					continue nextFile
 				}
+				addedFiles := 0
 			compareFile:
 				for otherKey, other := range files[partIdx] {
+					addedFiles++
+					if attempt > 0 && len(m.filled) == len(combineFilled) {
+						fmt.Println("Merging previous global data")
+						filled := 0
+						missing := 0
+						for i, v := range combineFilled {
+							if v == 1 {
+								m.filled[i] = 1
+								m.mapped[i] = combineShared[i]
+								filled++
+							} else {
+								missing++
+							}
+						}
+						fmt.Println("Missing", missing, "bytes. Filled", filled, "bytes.")
+						break
+					}
 					if key == otherKey {
 						continue
 					}
+
 					otherPart := getPartNum(other[0])
 					if part != otherPart {
 						fmt.Println("part ", part, " != other part", otherPart, other[0])
@@ -1076,16 +1112,6 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 					if m.objSize != otherM.objSize {
 						continue
 					}
-					var ok int
-					for i, filled := range otherM.filled[:m.size] {
-						if filled == 1 && m.filled[i] == 1 {
-							if m.mapped[i] != otherM.mapped[i] {
-								fmt.Println("Data mismatch at byte", i, "-  Disregarding version", otherKey)
-								continue compareFile
-							}
-							ok++
-						}
-					}
 
 					// If data+parity matches, combine.
 					if m.parity == otherM.parity && m.data == otherM.data {
@@ -1102,6 +1128,17 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						}
 					}
 
+					var ok int
+					for i, filled := range otherM.filled[:m.size] {
+						if filled == 1 && m.filled[i] == 1 {
+							if m.mapped[i] != otherM.mapped[i] {
+								fmt.Println("Data mismatch at byte", i, "-  Disregarding version", otherKey)
+								continue compareFile
+							}
+							ok++
+						}
+					}
+
 					fmt.Printf("Data overlaps (%d bytes). Combining with %q.\n", ok, otherKey)
 					for i := range otherM.filled {
 						if otherM.filled[i] == 1 {
@@ -1110,6 +1147,7 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						}
 					}
 				}
+
 				lastValid := 0
 				missing := 0
 				for i := range m.filled {
@@ -1130,7 +1168,7 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						if err != nil {
 							return err
 						}
-						split, err := rs.Split(m.mapped)
+						splitData, err := rs.Split(m.mapped)
 						if err != nil {
 							return err
 						}
@@ -1138,35 +1176,172 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 						if err != nil {
 							return err
 						}
-						ok := len(splitFilled)
-						for i, sh := range splitFilled {
-							for _, v := range sh {
+						// Fill padding...
+						padding := len(splitFilled[0])*k - len(m.filled)
+						for i := 0; i < padding; i++ {
+							arr := splitFilled[k-1]
+							arr[len(arr)-i-1] = 1
+						}
+
+						hasParity := 0
+						parityOK := make([]bool, m.shards)
+						for idx, sh := range v {
+							splitData[idx] = sh
+							if idx >= k && len(sh) > 0 {
+								parityOK[idx] = true
+								hasParity++
+								for i := range splitFilled[idx] {
+									splitFilled[idx][i] = 1
+								}
+							}
+						}
+
+						splitDataShards := make([]byte, len(splitFilled[0]))
+						for _, sh := range splitFilled {
+							for i, v := range sh {
+								splitDataShards[i] += v
+							}
+						}
+						var hist [256]int
+						for _, v := range splitDataShards {
+							hist[v]++
+						}
+
+						for _, v := range hist[m.data-hasParity : m.shards] {
+							if attempt > 0 {
+								break
+							}
+							if v == 0 {
+								continue
+							}
+							for i, v := range hist[:m.shards] {
+								if v > 0 {
+									if i < m.data {
+										fmt.Println("- Shards:", i, "of", m.data, "Bytes:", v, "Missing: ", v*(m.data-i+hasParity))
+									} else {
+										fmt.Println("+ Shards:", i, "of", m.data, "Bytes:", v, "Recovering: ", v*(m.data-i+hasParity))
+									}
+								}
+							}
+							fmt.Println("Attempting to reconstruct with partial shards")
+							offset := 0
+							startOffset := 0
+							shardConfig := make([]byte, k)
+							reconstructAbleConfig := false
+							shards := make([][]byte, m.shards)
+							for i := range shards {
+								shards[i] = make([]byte, 0, len(splitData[0]))
+							}
+							for offset < len(splitDataShards) {
+								newConfig := false
+								for shardIdx, shard := range splitFilled[:k] {
+									if shardConfig[shardIdx] != shard[offset] {
+										newConfig = true
+										break
+									}
+								}
+								if newConfig {
+									if offset > startOffset && reconstructAbleConfig {
+										reconPartial(shards, k, parityOK, splitData, startOffset, offset, rs, shardConfig, splitFilled)
+									}
+									// Update to new config and add current
+									valid := 0
+									for shardIdx, shard := range splitFilled[:k] {
+										shardConfig[shardIdx] = shard[offset]
+										valid += int(shard[offset])
+										if shard[offset] == 0 {
+											shards[shardIdx] = shards[shardIdx][:0]
+										} else {
+											shards[shardIdx] = append(shards[shardIdx][:0], splitData[shardIdx][offset])
+										}
+									}
+									reconstructAbleConfig = valid >= m.data-hasParity && valid < m.data
+									startOffset = offset
+									offset++
+									continue
+								}
+								for shardIdx, ok := range shardConfig {
+									if ok != 0 {
+										shards[shardIdx] = append(shards[shardIdx], splitData[shardIdx][offset])
+									}
+								}
+								offset++
+							}
+							if offset > startOffset && reconstructAbleConfig {
+								reconPartial(shards, k, parityOK, splitData, startOffset, offset, rs, shardConfig, splitFilled)
+							}
+
+							var buf bytes.Buffer
+							if err := rs.Join(&buf, splitFilled, m.size); err == nil {
+								m.filled = buf.Bytes()
+							}
+							buf = bytes.Buffer{}
+							if err := rs.Join(&buf, splitData, m.size); err == nil {
+								m.mapped = buf.Bytes()
+							}
+							for i, v := range m.filled {
 								if v == 0 {
-									split[i] = nil
-									ok--
+									m.mapped[i] = 0
+								}
+							}
+							break
+						}
+						ok := k
+						for i, sh := range splitFilled {
+							for j, v := range sh {
+								if v == 0 {
+									splitData[i] = nil
+									if i < k {
+										fmt.Println("Shard", i, "is missing data from offset", i*len(sh)+j)
+										ok--
+									}
 									break
 								}
 							}
 						}
-						hasParity := 0
-						for idx, sh := range v {
-							split[idx] = sh
-							if idx >= k && len(sh) > 0 {
-								hasParity++
+
+						missing = 0
+						lastValid = 0
+						for i := range m.filled {
+							if m.filled[i] == 1 {
+								lastValid = i
+							} else {
+								missing++
 							}
 						}
-						fmt.Printf("Have %d complete remapped data shards and %d complete parity shards. ", ok, hasParity)
+						fmt.Printf("Have %d complete remapped data shards and %d complete parity shards (%d bytes missing). ", ok, hasParity, missing)
 
-						if err := rs.ReconstructData(split); err == nil {
-							fmt.Println("Could reconstruct completely")
-							for i, data := range split[:k] {
+						if err := rs.ReconstructData(splitData); err == nil {
+							fmt.Println("Could reconstruct completely.")
+							for i, data := range splitData[:k] {
 								start := i * len(data)
 								copy(m.mapped[start:], data)
 							}
 							lastValid = m.size - 1
 							missing = 0
+							attempt = 2
+							wantMD5 = m.wantMD5
 						} else {
-							fmt.Println("Could NOT reconstruct:", err)
+							fmt.Println("Could NOT reconstruct:", err, " - Need", m.data, "shards.")
+							if attempt == 0 {
+								if len(combineShared) == 0 {
+									combineShared = make([]byte, len(m.mapped))
+									combineFilled = make([]byte, len(m.filled))
+								}
+								for i := range m.filled {
+									if m.filled[i] == 1 && combineFilled[i] == 0 {
+										combineShared[i] = m.mapped[i]
+										combineFilled[i] = 1
+									}
+								}
+								combineFilledBlocks[block] = combineFilled
+								combineSharedBlocks[block] = combineShared
+								fmt.Println("Retrying with merged data")
+								if addedFiles >= len(files[partIdx]) {
+									attempt++
+									goto nextAttempt
+								}
+							}
 						}
 					}
 				}
@@ -1182,7 +1357,7 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 					if len(combined) != m.partSize {
 						fmt.Println("Combined size mismatch. Expected", m.partSize, "got", len(combined))
 					}
-					fmt.Println("Reached block", block+1, "of", m.blocks+1, "for", key, ". Done.")
+					fmt.Println("Reached block", block+1, "of", m.blocks+1, "for", key, "Done.")
 					break
 				}
 			}
@@ -1195,16 +1370,31 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 			}
 			if missingAll > 0 {
 				out += ".incomplete"
-				fmt.Println(missingAll, "bytes missing. Truncating", len(combined)-lastValidAll-1, "from end.")
+				fmt.Println(missingAll, "bytes missing.")
 			} else {
-				out += ".complete"
-				fmt.Println("No bytes missing.")
+				if wantMD5 != "" {
+					sum := md5.Sum(combined)
+					gotMD5 := hex.EncodeToString(sum[:])
+					if gotMD5 != wantMD5 {
+						fmt.Println("MD5 mismatch. Expected", wantMD5, "got", gotMD5)
+						out += ".mismatch"
+					} else {
+						fmt.Println("MD5 verified.")
+						out = fmt.Sprintf("verified/%s", baseName)
+					}
+				} else {
+					out = fmt.Sprintf("complete/%s.%05d", baseName, part)
+					fmt.Println("No bytes missing.")
+				}
 			}
 			if missingAll == 0 {
 				exportedSizes[len(combined)] = true
 			}
-			combined = combined[:lastValidAll+1]
-			err := os.WriteFile(out, combined, os.ModePerm)
+			err := os.MkdirAll(filepath.Dir(out), os.ModePerm)
+			if err != nil {
+				return err
+			}
+			err = os.WriteFile(out, combined, os.ModePerm)
 			if err != nil {
 				return err
 			}
@@ -1214,6 +1404,46 @@ func combineCrossVer(all map[string][]string, baseName string) error {
 	return nil
 }
 
+func reconPartial(shards [][]byte, k int, parityOK []bool, splitData [][]byte, startOffset int, offset int, rs reedsolomon.Encoder, shardConfig []byte, splitFilled [][]byte) {
+	// Add parity
+	for i := range shards[k:] {
+		shards[i+k] = nil
+		if parityOK[i+k] {
+			shards[i+k] = splitData[i+k][startOffset:offset]
+		}
+	}
+	// Reconstruct with current config.
+	if err := rs.ReconstructData(shards); err != nil {
+		panic(fmt.Sprintln("Internal error, could NOT partially reconstruct:", err))
+	}
+	// Copy reconstructed data back.
+	verified := 0
+	reconstructed := 0
+	for shardsIdx, ok := range shardConfig {
+		if ok == 0 {
+			copy(splitData[shardsIdx][startOffset:], shards[shardsIdx])
+			for i := range shards[shardsIdx] {
+				if splitFilled[shardsIdx][startOffset+i] == 1 {
+					fmt.Println("Internal error: Found filled data at", startOffset+i)
+				}
+				splitFilled[shardsIdx][startOffset+i] = 1
+			}
+			reconstructed += len(shards[shardsIdx])
+		} else {
+			for i := range shards[shardsIdx] {
+				if splitFilled[shardsIdx][startOffset+i] == 0 {
+					fmt.Println("Internal error: Expected filled data at", startOffset+i)
+				}
+				if splitData[shardsIdx][startOffset+i] != shards[shardsIdx][i] {
+					fmt.Println("Internal error: Mismatch at", startOffset+i)
+				}
+				verified++
+			}
+		}
+	}
+	fmt.Println("Reconstructed", reconstructed, "bytes and verified", verified, "bytes of partial shard with config", shardConfig)
+}
+
 // bitrot returns a shard beginning at startOffset after doing bitrot checks.
 func bitrot(val []byte, startOffset, shardSize int) ([]byte, error) {
 	var res []byte

docs/metrics/prometheus/grafana/node/minio-node.json

@@ -394,32 +394,7 @@
           },
           "unit": "none"
         },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "[play.min.io:9000:/disk4/data]"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
+        "overrides": []
       },
       "gridPos": {
         "h": 5,

docs/metrics/v3.md

@@ -42,6 +42,14 @@ These are metrics about the minio audit functionality
 |----------|----------------------------------------|
 | `/audit` | Metrics related to audit functionality |
 
+### ILM metrics
+
+These are metrics about the minio ILM functionality
+
+| Path   | Description                          |
+|--------|--------------------------------------|
+| `/ilm` | Metrics related to ILM functionality |
+
 ### Logger webhook metrics
 
 These are metrics about the minio logger webhooks
@@ -377,3 +385,13 @@ The standard metrics group for GoCollector is not shown below.
 | `minio_scanner_last_activity_seconds`      | `gauge`   | Time elapsed (in seconds) since last scan activity         | `server` |
 | `minio_scanner_objects_scanned`            | `counter` | Total number of unique objects scanned since server start  | `server` |
 | `minio_scanner_versions_scanned`           | `counter` | Total number of object versions scanned since server start | `server` |
+
+### `/ilm`
+
+| Name                                                  | Type      | Help                                                                       | Labels   |
+|-------------------------------------------------------|-----------|----------------------------------------------------------------------------|----------|
+| `minio_cluster_ilm_expiry_pending_tasks`              | `gauge`   | Number of pending ILM expiry tasks in the queue                            | `server` |
+| `minio_cluster_ilm_transition_active_tasks`           | `gauge`   | Number of active ILM transition tasks                                      | `server` |
+| `minio_cluster_ilm_transition_pending_tasks`          | `gauge`   | Number of pending ILM transition tasks in the queue                        | `server` |
+| `minio_cluster_ilm_transition_missed_immediate_tasks` | `counter` | Number of missed immediate ILM transition tasks                            | `server` |
+| `minio_cluster_ilm_versions_scanned`                  | `counter` | Total number of object versions checked for ILM actions since server start | `server` |

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2024-05-27T19-17-46Z
+  image: quay.io/minio/minio:RELEASE.2024-06-04T19-20-08Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -6,16 +6,16 @@ require (
 	cloud.google.com/go/storage v1.41.0
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Azure/go-autorest/autorest v0.11.29
-	github.com/Azure/go-autorest/autorest/adal v0.9.23
+	github.com/Azure/go-autorest/autorest/adal v0.9.24
 	github.com/IBM/sarama v1.43.2
 	github.com/alecthomas/participle v0.7.1
 	github.com/bcicen/jstream v1.0.1
-	github.com/beevik/ntp v1.4.2
+	github.com/beevik/ntp v1.4.3
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coredns/coredns v1.11.3
-	github.com/coreos/go-oidc v2.2.1+incompatible
+	github.com/coreos/go-oidc/v3 v3.10.0
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/cosnicolaou/pbzip2 v1.0.3
 	github.com/dchest/siphash v1.2.3
@@ -52,7 +52,7 @@ require (
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kms-go/kes v0.3.0
 	github.com/minio/kms-go/kms v0.4.0
-	github.com/minio/madmin-go/v3 v3.0.52
+	github.com/minio/madmin-go/v3 v3.0.55-0.20240603092915-420a67132c32
 	github.com/minio/minio-go/v7 v7.0.70
 	github.com/minio/mux v1.9.0
 	github.com/minio/pkg/v3 v3.0.1
@@ -74,31 +74,31 @@ require (
 	github.com/pkg/xattr v0.4.9
 	github.com/prometheus/client_golang v1.19.1
 	github.com/prometheus/client_model v0.6.1
-	github.com/prometheus/common v0.53.0
-	github.com/prometheus/procfs v0.15.0
+	github.com/prometheus/common v0.54.0
+	github.com/prometheus/procfs v0.15.1
 	github.com/puzpuzpuz/xsync/v3 v3.1.0
 	github.com/rabbitmq/amqp091-go v1.10.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/rs/cors v1.11.0
 	github.com/secure-io/sio-go v0.3.1
-	github.com/shirou/gopsutil/v3 v3.24.4
+	github.com/shirou/gopsutil/v3 v3.24.5
 	github.com/tidwall/gjson v1.17.1
 	github.com/tinylib/msgp v1.1.9
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/xdg/scram v1.0.5
 	github.com/zeebo/xxh3 v1.0.2
-	go.etcd.io/etcd/api/v3 v3.5.13
-	go.etcd.io/etcd/client/v3 v3.5.13
+	go.etcd.io/etcd/api/v3 v3.5.14
+	go.etcd.io/etcd/client/v3 v3.5.14
 	go.uber.org/atomic v1.11.0
 	go.uber.org/zap v1.27.0
 	goftp.io/server/v2 v2.0.1
-	golang.org/x/crypto v0.23.0
-	golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d
-	golang.org/x/oauth2 v0.20.0
-	golang.org/x/sys v0.20.0
-	golang.org/x/term v0.20.0
+	golang.org/x/crypto v0.24.0
+	golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc
+	golang.org/x/oauth2 v0.21.0
+	golang.org/x/sys v0.21.0
+	golang.org/x/term v0.21.0
 	golang.org/x/time v0.5.0
-	google.golang.org/api v0.181.0
+	google.golang.org/api v0.182.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -107,7 +107,7 @@ require (
 	aead.dev/mem v0.2.0 // indirect
 	aead.dev/minisign v0.3.0 // indirect
 	cloud.google.com/go v0.114.0 // indirect
-	cloud.google.com/go/auth v0.4.2 // indirect
+	cloud.google.com/go/auth v0.5.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/iam v1.1.8 // indirect
@@ -126,10 +126,10 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/charmbracelet/bubbles v0.18.0 // indirect
-	github.com/charmbracelet/bubbletea v0.26.3 // indirect
+	github.com/charmbracelet/bubbletea v0.26.4 // indirect
 	github.com/charmbracelet/lipgloss v0.11.0 // indirect
-	github.com/charmbracelet/x/ansi v0.1.1 // indirect
-	github.com/charmbracelet/x/input v0.1.1 // indirect
+	github.com/charmbracelet/x/ansi v0.1.2 // indirect
+	github.com/charmbracelet/x/input v0.1.2 // indirect
 	github.com/charmbracelet/x/term v0.1.1 // indirect
 	github.com/charmbracelet/x/windows v0.1.2 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
@@ -144,6 +144,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/frankban/quicktest v1.14.4 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -200,9 +201,9 @@ require (
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/minio/colorjson v1.0.7 // indirect
+	github.com/minio/colorjson v1.0.8 // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20240524090849-a8fdcbe7cb2f // indirect
+	github.com/minio/mc v0.0.0-20240601150335-3cfa8642fdab // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/pkg/v2 v2.0.19 // indirect
 	github.com/minio/websocket v1.6.0 // indirect
@@ -223,7 +224,6 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/posener/complete v1.2.3 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
-	github.com/pquerna/cachecontrol v0.2.0 // indirect
 	github.com/prometheus/prom2json v1.3.3 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rjeczalik/notify v0.9.3 // indirect
@@ -239,7 +239,7 @@ require (
 	github.com/xdg/stringprep v1.0.3 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
 	go.mongodb.org/mongo-driver v1.15.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect
@@ -248,16 +248,15 @@ require (
 	go.opentelemetry.io/otel/metric v1.27.0 // indirect
 	go.opentelemetry.io/otel/trace v1.27.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	golang.org/x/tools v0.21.0 // indirect
-	google.golang.org/genproto v0.0.0-20240521202816-d264139d666e // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	google.golang.org/genproto v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/grpc v1.64.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 )

go.sum

@@ -6,8 +6,8 @@ aead.dev/minisign v0.3.0/go.mod h1:NLvG3Uoq3skkRMDuc3YHpWUTMTrSExqm+Ij73W13F6Y=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY=
 cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E=
-cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg=
-cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc=
+cloud.google.com/go/auth v0.5.1 h1:0QNO7VThG54LUzKiQxv8C6x1YX7lUrzlAa1nVLF8CIw=
+cloud.google.com/go/auth v0.5.1/go.mod h1:vbZT8GjzDf3AVqCcQmqeeM32U9HBFc32vVVAbwDsa6s=
 cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4=
 cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q=
 cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
@@ -28,8 +28,8 @@ github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/
 github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk=
-github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8=
-github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c=
+github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4=
+github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
@@ -73,8 +73,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck=
 github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ=
-github.com/beevik/ntp v1.4.2 h1:cjYhZqczanf6br/ocViahE75ipj7CmKQAh7fSBaCNK4=
-github.com/beevik/ntp v1.4.2/go.mod h1:zkATLTt8VUZuOfYX2KgOnir4yvtAxWbnUUA24umXFnc=
+github.com/beevik/ntp v1.4.3 h1:PlbTvE5NNy4QHmA4Mg57n7mcFTmr1W1j3gcK7L1lqho=
+github.com/beevik/ntp v1.4.3/go.mod h1:Unr8Zg+2dRn7d8bHFuehIMSvvUYssHMxW3Q5Nx4RW5Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -87,14 +87,14 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0=
 github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw=
-github.com/charmbracelet/bubbletea v0.26.3 h1:iXyGvI+FfOWqkB2V07m1DF3xxQijxjY2j8PqiXYqasg=
-github.com/charmbracelet/bubbletea v0.26.3/go.mod h1:bpZHfDHTYJC5g+FBK+ptJRCQotRC+Dhh3AoMxa/2+3Q=
+github.com/charmbracelet/bubbletea v0.26.4 h1:2gDkkzLZaTjMl/dQBpNVtnvcCxsh/FCkimep7FC9c40=
+github.com/charmbracelet/bubbletea v0.26.4/go.mod h1:P+r+RRA5qtI1DOHNFn0otoNwB4rn+zNAzSj/EXz6xU0=
 github.com/charmbracelet/lipgloss v0.11.0 h1:UoAcbQ6Qml8hDwSWs0Y1cB5TEQuZkDPH/ZqwWWYTG4g=
 github.com/charmbracelet/lipgloss v0.11.0/go.mod h1:1UdRTH9gYgpcdNN5oBtjbu/IzNKtzVtb7sqN1t9LNn8=
-github.com/charmbracelet/x/ansi v0.1.1 h1:CGAduulr6egay/YVbGc8Hsu8deMg1xZ/bkaXTPi1JDk=
-github.com/charmbracelet/x/ansi v0.1.1/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
-github.com/charmbracelet/x/input v0.1.1 h1:YDOJaTUKCqtGnq9PHzx3pkkl4pXDOANUHmhH3DqMtM4=
-github.com/charmbracelet/x/input v0.1.1/go.mod h1:jvdTVUnNWj/RD6hjC4FsoB0SeZCJ2ZBkiuFP9zXvZI0=
+github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY=
+github.com/charmbracelet/x/ansi v0.1.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
+github.com/charmbracelet/x/input v0.1.2 h1:QJAZr33eOhDowkkEQ24rsJy4Llxlm+fRDf/cQrmqJa0=
+github.com/charmbracelet/x/input v0.1.2/go.mod h1:LGBim0maUY4Pitjn/4fHnuXb4KirU3DODsyuHuXdOyA=
 github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI=
 github.com/charmbracelet/x/term v0.1.1/go.mod h1:wB1fHt5ECsu3mXYusyzcngVWWlu1KKUmmLhfgr/Flxw=
 github.com/charmbracelet/x/windows v0.1.2 h1:Iumiwq2G+BRmgoayww/qfcvof7W/3uLoelhxojXlRWg=
@@ -115,8 +115,8 @@ github.com/coredns/coredns v1.11.3 h1:8RjnpZc42db5th84/QJKH2i137ecJdzZK1HJwhetSP
 github.com/coredns/coredns v1.11.3/go.mod h1:lqFkDsHjEUdY7LJ75Nib3lwqJGip6ewWOqNIf8OavIQ=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
-github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk=
-github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU=
+github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
@@ -178,6 +178,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-asn1-ber/asn1-ber v1.5.7 h1:DTX+lbVTWaTw1hQ+PbZPlnDZPEIs0SS/GCZAl535dDk=
 github.com/go-asn1-ber/asn1-ber v1.5.7/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
+github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-ldap/ldap/v3 v3.4.8 h1:loKJyspcRezt2Q3ZRMq2p/0v8iOurlmeXDPw6fikSvQ=
@@ -264,7 +266,6 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -403,7 +404,6 @@ github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw
 github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
-github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -436,8 +436,8 @@ github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
 github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
 github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
 github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
-github.com/minio/colorjson v1.0.7 h1:n69M42mIuQHdzbsxlmwji1zxDypaw4o39rHjAmX4Dh4=
-github.com/minio/colorjson v1.0.7/go.mod h1:9LGM5yybI+GuhSbuzAerbSgvFb4j8ux9NzyONR+NrAY=
+github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg=
+github.com/minio/colorjson v1.0.8/go.mod h1:wrs39G/4kqNlGjwqHvPlAnXuc2tlPszo6JKdSBCLN8w=
 github.com/minio/console v1.4.1 h1:P7hgyQi+36aYH90WPME3d/eLJ+a1jxnfhwxLjUOe9kY=
 github.com/minio/console v1.4.1/go.mod h1:JyqeznIlKwgSx2Usz4CNq0i9WlDMJF75m8lbPV38p4I=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
@@ -454,10 +454,10 @@ github.com/minio/kms-go/kes v0.3.0 h1:SU8VGVM/Hk9w1OiSby3OatkcojooUqIdDHl6dtM6Nk
 github.com/minio/kms-go/kes v0.3.0/go.mod h1:w6DeVT878qEOU3nUrYVy1WOT5H1Ig9hbDIh698NYJKY=
 github.com/minio/kms-go/kms v0.4.0 h1:cLPZceEp+05xHotVBaeFJrgL7JcXM4lBy6PU0idkE7I=
 github.com/minio/kms-go/kms v0.4.0/go.mod h1:q12CehiIy2qgBnDKq6Q7wmPi2PHSyRVug5DKp0HAVeE=
-github.com/minio/madmin-go/v3 v3.0.52 h1:X2zoNfEkrgql9Hlal6Nfw7pAqLAM47yZy4i7yb2bUVk=
-github.com/minio/madmin-go/v3 v3.0.52/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw=
-github.com/minio/mc v0.0.0-20240524090849-a8fdcbe7cb2f h1:UPxKkqMqyHNb+68hTq+PDIUYAqAbbplUujEyP5Ez9rs=
-github.com/minio/mc v0.0.0-20240524090849-a8fdcbe7cb2f/go.mod h1:d/mgZMbu2yRNyYOwVqvRBV25pKMfAz7fijObWSXkqpA=
+github.com/minio/madmin-go/v3 v3.0.55-0.20240603092915-420a67132c32 h1:9se7/S4AlN2k/B1E7A8m1m07DM3p0JnIOzVhDuAV2PI=
+github.com/minio/madmin-go/v3 v3.0.55-0.20240603092915-420a67132c32/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw=
+github.com/minio/mc v0.0.0-20240601150335-3cfa8642fdab h1:qpM4fcVhes6aL5v4kImPydv/4Kj61TtUbYEafDozAyk=
+github.com/minio/mc v0.0.0-20240601150335-3cfa8642fdab/go.mod h1:d/mgZMbu2yRNyYOwVqvRBV25pKMfAz7fijObWSXkqpA=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
@@ -558,11 +558,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
-github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k=
-github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -578,15 +575,15 @@ github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQy
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
-github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE=
-github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U=
+github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8=
+github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek=
-github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/prometheus/prom2json v1.3.3 h1:IYfSMiZ7sSOfliBoo89PcufjWO4eAR0gznGcETyaUgo=
 github.com/prometheus/prom2json v1.3.3/go.mod h1:Pv4yIPktEkK7btWsrUTWDDDrnpUrAELaOCj+oFwlgmc=
 github.com/puzpuzpuz/xsync/v3 v3.1.0 h1:EewKT7/LNac5SLiEblJeUu8z5eERHrmRLnMQL2d7qX4=
@@ -614,8 +611,8 @@ github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6o
 github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=
 github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=
-github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU=
-github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8=
+github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
+github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -658,10 +655,8 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso
 github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
 github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU=
 github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k=
-github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
 github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
-github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
 github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY=
 github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -690,12 +685,12 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=
-go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c=
-go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg=
-go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8=
-go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js=
-go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI=
+go.etcd.io/etcd/api/v3 v3.5.14 h1:vHObSCxyB9zlF60w7qzAdTcGaglbJOpSj1Xj9+WGxq0=
+go.etcd.io/etcd/api/v3 v3.5.14/go.mod h1:BmtWcRlQvwa1h3G2jvKYwIQy4PkHlDej5t7uLMUdJUU=
+go.etcd.io/etcd/client/pkg/v3 v3.5.14 h1:SaNH6Y+rVEdxfpA2Jr5wkEvN6Zykme5+YnbCkxvuWxQ=
+go.etcd.io/etcd/client/pkg/v3 v3.5.14/go.mod h1:8uMgAokyG1czCtIdsq+AGyYQMvpIKnSvPjFMunkgeZI=
+go.etcd.io/etcd/client/v3 v3.5.14 h1:CWfRs4FDaDoSz81giL7zPpZH2Z35tbOrAJkkjMqOupg=
+go.etcd.io/etcd/client/v3 v3.5.14/go.mod h1:k3XfdV/VIHy/97rqWjoUzrj9tk7GgJGH9J8L4dNXmAk=
 go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
 go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
@@ -742,13 +737,14 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d h1:N0hmiNbwsSNwHBAvR3QB5w25pUwH4tK0Y/RltD1j1h4=
-golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
+golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg=
+golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -756,8 +752,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -788,8 +784,8 @@ golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
-golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
+golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -840,12 +836,11 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -853,10 +848,11 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -867,8 +863,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
@@ -885,27 +881,27 @@ golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
-golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4=
-google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k=
+google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE=
+google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20240521202816-d264139d666e h1:axIBUGXSVho2zB+3tJj8l9Qvm/El5vVYPYqhGA5PmJM=
-google.golang.org/genproto v0.0.0-20240521202816-d264139d666e/go.mod h1:gOvX/2dWTqh+u3+IHjFeCxinlz5AZ5qhOufbQPub/dE=
-google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0=
-google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
+google.golang.org/genproto v0.0.0-20240528184218-531527333157 h1:u7WMYrIrVvs0TF5yaKwKNbcJyySYf+HAIFXxWltJOXE=
+google.golang.org/genproto v0.0.0-20240528184218-531527333157/go.mod h1:ubQlAQnzejB8uZzszhrTCU2Fyp6Vi7ZE5nn0c3W8+qQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
@@ -932,8 +928,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
-gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/config/identity/openid/jwt.go

@@ -133,7 +133,7 @@ const (
 )
 
 // Validate - validates the id_token.
-func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken, dsecs string, claims jwtgo.MapClaims) error {
+func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken, dsecs string, claims map[string]interface{}) error {
 	jp := new(jwtgo.Parser)
 	jp.ValidMethods = []string{
 		"RS256", "RS384", "RS512",
@@ -156,14 +156,15 @@ func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken,
 		return fmt.Errorf("Role %s does not exist", arn)
 	}
 
-	jwtToken, err := jp.ParseWithClaims(token, &claims, keyFuncCallback)
+	mclaims := jwtgo.MapClaims(claims)
+	jwtToken, err := jp.ParseWithClaims(token, &mclaims, keyFuncCallback)
 	if err != nil {
 		// Re-populate the public key in-case the JWKS
 		// pubkeys are refreshed
 		if err = r.PopulatePublicKey(arn); err != nil {
 			return err
 		}
-		jwtToken, err = jwtgo.ParseWithClaims(token, &claims, keyFuncCallback)
+		jwtToken, err = jwtgo.ParseWithClaims(token, &mclaims, keyFuncCallback)
 		if err != nil {
 			return err
 		}
@@ -173,11 +174,11 @@ func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken,
 		return ErrTokenExpired
 	}
 
-	if err = updateClaimsExpiry(dsecs, claims); err != nil {
+	if err = updateClaimsExpiry(dsecs, mclaims); err != nil {
 		return err
 	}
 
-	if err = r.updateUserinfoClaims(ctx, arn, accessToken, claims); err != nil {
+	if err = r.updateUserinfoClaims(ctx, arn, accessToken, mclaims); err != nil {
 		return err
 	}
 
@@ -190,7 +191,7 @@ func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken,
 	// array of case sensitive strings. In the common special case
 	// when there is one audience, the aud value MAY be a single
 	// case sensitive
-	audValues, ok := policy.GetValuesFromClaims(claims, audClaim)
+	audValues, ok := policy.GetValuesFromClaims(mclaims, audClaim)
 	if !ok {
 		return errors.New("STS JWT Token has `aud` claim invalid, `aud` must match configured OpenID Client ID")
 	}
@@ -204,7 +205,7 @@ func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken,
 		// be included even when the authorized party is the same
 		// as the sole audience. The azp value is a case sensitive
 		// string containing a StringOrURI value
-		azpValues, ok := policy.GetValuesFromClaims(claims, azpClaim)
+		azpValues, ok := policy.GetValuesFromClaims(mclaims, azpClaim)
 		if !ok {
 			return errors.New("STS JWT Token has `azp` claim invalid, `azp` must match configured OpenID Client ID")
 		}

internal/config/ilm/help.go

@@ -33,9 +33,9 @@ var (
 		return config.DefaultHelpPostfix(DefaultKVS, key)
 	}
 
-	// HelpILM holds configuration keys and their default values for the ILM
+	// Help holds configuration keys and their default values for the ILM
 	// subsystem
-	HelpILM = config.HelpKVS{
+	Help = config.HelpKVS{
 		config.HelpKV{
 			Key:         transitionWorkers,
 			Type:        "number",

internal/event/target/kafka.go

@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -331,6 +332,10 @@ func (target *KafkaTarget) init() error {
 }
 
 func (target *KafkaTarget) initKafka() error {
+	if os.Getenv("_MINIO_KAFKA_DEBUG") != "" {
+		sarama.DebugLogger = log.Default()
+	}
+
 	args := target.args
 
 	config := sarama.NewConfig()

internal/grid/connection.go

@@ -122,6 +122,10 @@ type Connection struct {
 	outgoingBytes func(n int64) // Record outgoing bytes.
 	trace         *tracer       // tracer for this connection.
 	baseFlags     Flags
+	outBytes      atomic.Int64
+	inBytes       atomic.Int64
+	inMessages    atomic.Int64
+	outMessages   atomic.Int64
 
 	// For testing only
 	debugInConn  net.Conn
@@ -232,13 +236,25 @@ func newConnection(o connectionParams) *Connection {
 		clientPingInterval: clientPingInterval,
 		connPingInterval:   connPingInterval,
 		tlsConfig:          o.tlsConfig,
-		incomingBytes:      o.incomingBytes,
-		outgoingBytes:      o.outgoingBytes,
 	}
 	if debugPrint {
 		// Random Mux ID
 		c.NextID = rand.Uint64()
 	}
+
+	// Record per connection stats.
+	c.outgoingBytes = func(n int64) {
+		if o.outgoingBytes != nil {
+			o.outgoingBytes(n)
+		}
+		c.outBytes.Add(n)
+	}
+	c.incomingBytes = func(n int64) {
+		if o.incomingBytes != nil {
+			o.incomingBytes(n)
+		}
+		c.inBytes.Add(n)
+	}
 	if !strings.HasPrefix(o.remote, "https://") && !strings.HasPrefix(o.remote, "wss://") {
 		c.baseFlags |= FlagCRCxxh3
 	}
@@ -995,11 +1011,13 @@ func (c *Connection) handleMessages(ctx context.Context, conn net.Conn) {
 				fmt.Printf("%s Got msg: %v\n", c.Local, m)
 			}
 			if m.Op != OpMerged {
+				c.inMessages.Add(1)
 				c.handleMsg(ctx, m, subID)
 				continue
 			}
 			// Handle merged messages.
 			messages := int(m.Seq)
+			c.inMessages.Add(int64(messages))
 			for i := 0; i < messages; i++ {
 				if atomic.LoadUint32((*uint32)(&c.state)) != StateConnected {
 					cancel(ErrDisconnected)
@@ -1100,6 +1118,11 @@ func (c *Connection) handleMessages(ctx context.Context, conn net.Conn) {
 			queueSize += len(toSend)
 			continue
 		}
+		c.outMessages.Add(int64(len(queue) + 1))
+		if c.outgoingBytes != nil {
+			c.outgoingBytes(int64(len(toSend) + queueSize))
+		}
+
 		c.connChange.L.Lock()
 		for {
 			state := c.State()
@@ -1578,11 +1601,28 @@ func (c *Connection) State() State {
 }
 
 // Stats returns the current connection stats.
-func (c *Connection) Stats() ConnectionStats {
-	return ConnectionStats{
-		IncomingStreams: c.inStream.Size(),
-		OutgoingStreams: c.outgoing.Size(),
+func (c *Connection) Stats() madmin.RPCMetrics {
+	conn := 0
+	if c.State() == StateConnected {
+		conn++
 	}
+	m := madmin.RPCMetrics{
+		CollectedAt:      time.Now(),
+		Connected:        conn,
+		Disconnected:     1 - conn,
+		IncomingStreams:  c.inStream.Size(),
+		OutgoingStreams:  c.outgoing.Size(),
+		IncomingBytes:    c.inBytes.Load(),
+		OutgoingBytes:    c.outBytes.Load(),
+		IncomingMessages: c.inMessages.Load(),
+		OutgoingMessages: c.outMessages.Load(),
+		OutQueue:         len(c.outQueue),
+		LastPongTime:     time.Unix(c.LastPong, 0).UTC(),
+	}
+	m.ByDestination = map[string]madmin.RPCMetrics{
+		c.Remote: m,
+	}
+	return m
 }
 
 func (c *Connection) debugMsg(d debugMsg, args ...any) {

internal/grid/manager.go

@@ -334,3 +334,13 @@ func (m *Manager) debugMsg(d debugMsg, args ...any) {
 		c.debugMsg(d, args...)
 	}
 }
+
+// ConnStats returns the connection statistics for all connections.
+func (m *Manager) ConnStats() madmin.RPCMetrics {
+	var res madmin.RPCMetrics
+	for _, c := range m.targets {
+		t := c.Stats()
+		res.Merge(&t)
+	}
+	return res
+}

internal/grid/stats.go

@@ -1,24 +0,0 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package grid
-
-// ConnectionStats contains connection statistics.
-type ConnectionStats struct {
-	OutgoingStreams int
-	IncomingStreams int
-}

internal/grid/trace.go

@@ -105,6 +105,7 @@ func (c *muxClient) traceRoundtrip(ctx context.Context, t *tracer, h HandlerID,
 		Duration:  end.Sub(start),
 		Path:      t.Subroute,
 		Error:     errString,
+		Bytes:     int64(len(req) + len(resp)),
 		HTTP: &madmin.TraceHTTPStats{
 			ReqInfo: madmin.TraceRequestInfo{
 				Time:    start,

internal/logger/target/http/http.go

@@ -193,7 +193,20 @@ func (h *Target) initDiskStore(ctx context.Context) (err error) {
 	h.storeCtxCancel = cancel
 	h.lastStarted = time.Now()
 	go h.startQueueProcessor(ctx, true)
+
+	queueStore := store.NewQueueStore[interface{}](
+		filepath.Join(h.config.QueueDir, h.Name()),
+		uint64(h.config.QueueSize),
+		httpLoggerExtension,
+	)
+
+	if err := queueStore.Open(); err != nil {
+		return fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
+	}
+
+	h.store = queueStore
 	store.StreamItems(h.store, h, ctx.Done(), h.config.LogOnceIf)
+
 	return nil
 }
 
@@ -314,10 +327,7 @@ func (h *Target) startQueueProcessor(ctx context.Context, mainWorker bool) {
 	enc := jsoniter.ConfigCompatibleWithStandardLibrary.NewEncoder(buf)
 	defer bytebufferpool.Put(buf)
 
-	isDirQueue := false
-	if h.config.QueueDir != "" {
-		isDirQueue = true
-	}
+	isDirQueue := h.config.QueueDir != ""
 
 	// globalBuffer is always created or adjusted
 	// before this method is launched.
@@ -504,23 +514,6 @@ func New(config Config) (*Target, error) {
 	}
 
 	h.client = &http.Client{Transport: h.config.Transport}
-
-	if h.config.QueueDir != "" {
-
-		queueStore := store.NewQueueStore[interface{}](
-			filepath.Join(h.config.QueueDir, h.Name()),
-			uint64(h.config.QueueSize),
-			httpLoggerExtension,
-		)
-
-		if err := queueStore.Open(); err != nil {
-			return h, fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
-		}
-
-		h.store = queueStore
-
-	}
-
 	return h, nil
 }
 

internal/logger/target/kafka/kafka.go

@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"sync"
@@ -168,9 +169,8 @@ func (h *Target) Init(ctx context.Context) error {
 }
 
 func (h *Target) initQueueStore(ctx context.Context) (err error) {
-	var queueStore store.Store[interface{}]
 	queueDir := filepath.Join(h.kconfig.QueueDir, h.Name())
-	queueStore = store.NewQueueStore[interface{}](queueDir, uint64(h.kconfig.QueueSize), kafkaLoggerExtension)
+	queueStore := store.NewQueueStore[interface{}](queueDir, uint64(h.kconfig.QueueSize), kafkaLoggerExtension)
 	if err = queueStore.Open(); err != nil {
 		return fmt.Errorf("unable to initialize the queue store of %s webhook: %w", h.Name(), err)
 	}
@@ -234,6 +234,10 @@ func (h *Target) send(entry interface{}) error {
 
 // Init initialize kafka target
 func (h *Target) init() error {
+	if os.Getenv("_MINIO_KAFKA_DEBUG") != "" {
+		sarama.DebugLogger = log.Default()
+	}
+
 	sconfig := sarama.NewConfig()
 	if h.kconfig.Version != "" {
 		kafkaVersion, err := sarama.ParseKafkaVersion(h.kconfig.Version)

internal/logger/targets.go

@@ -45,16 +45,14 @@ type Target interface {
 
 type targetsList struct {
 	list []Target
-	mu   *sync.RWMutex
+	mu   sync.RWMutex
 }
 
 func newTargetsList() *targetsList {
-	return &targetsList{
-		mu: &sync.RWMutex{},
-	}
+	return &targetsList{}
 }
 
-func (tl targetsList) get() []Target {
+func (tl *targetsList) get() []Target {
 	tl.mu.RLock()
 	defer tl.mu.RUnlock()