replication: Pass metadata timestamps in CopyObject call (#18647)

Regression from #18285. CopyObject options were inheriting source MTime
for metadata timestamps if unspecified, removing this prevented metadata
updates from being applied on target.

.github/workflows/replication.yaml

@@ -36,6 +36,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-decom
 
+      - name: Test Config File
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-configfile
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.github/workflows/vulncheck.yml

@@ -3,11 +3,10 @@ on:
   pull_request:
     branches:
       - master
-      - next
+
   push:
     branches:
       - master
-      - next
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -22,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.4
+          go-version: 1.21.5
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.golangci.yml

@@ -29,5 +29,8 @@ linters:
 issues:
   exclude-use-default: false
   exclude:
+    - "empty-block:"
+    - "unused-parameter:"
+    - "dot-imports:"
     - should have a package comment
     - error strings should not be capitalized or end with punctuation or a newline

Dockerfile.hotfix

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \

Dockerfile.release

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \

Dockerfile.release.fips

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \

Dockerfile.release.old_cpu

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \

Makefile

@@ -8,7 +8,6 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "quay.io/minio/minio:$(VERSION)"
 
-GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
 GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
 
@@ -23,7 +22,7 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
 	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
@@ -60,6 +59,9 @@ test-decom: install-race
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
 	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
+test-configfile: install-race
+	@env bash $(PWD)/docs/distributed/distributed-from-config-file.sh
+
 test-upgrade: install-race
 	@echo "Running minio upgrade tests"
 	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)

cmd/admin-handlers-pools.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/minio/minio/internal/logger"
@@ -66,16 +67,28 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
 
 	pools := strings.Split(v, ",")
 	poolIndices := make([]int, 0, len(pools))
 
 	for _, pool := range pools {
-		idx := globalEndpoints.GetPoolIdx(pool)
-		if idx == -1 {
-			// We didn't find any matching pools, invalid input
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
-			return
+		var idx int
+		if byID {
+			var err error
+			idx, err = strconv.Atoi(pool)
+			if err != nil {
+				// We didn't find any matching pools, invalid input
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+				return
+			}
+		} else {
+			idx = globalEndpoints.GetPoolIdx(pool)
+			if idx == -1 {
+				// We didn't find any matching pools, invalid input
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+				return
+			}
 		}
 		var pool *erasureSets
 		for pidx := range z.serverPools {
@@ -132,8 +145,17 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		// We didn't find any matching pools, invalid input
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
@@ -178,8 +200,17 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
 		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)

cmd/admin-handlers.go

@@ -2572,7 +2572,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for _, tgt := range globalNotifyTargetList.Targets() {
+	for _, tgt := range globalEventNotifier.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()

cmd/admin-heal-ops.go

@@ -810,7 +810,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -867,7 +867,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 
 	if !h.settings.Recursive {
 		if h.object != "" {
-			if err := h.healObject(bucket, h.object, ""); err != nil {
+			if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
 				return err
 			}
 		}
@@ -882,7 +882,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 }
 
 // healObject - heal the given object and record result
-func (h *healSequence) healObject(bucket, object, versionID string) error {
+func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/batch-expire.go

@@ -33,9 +33,9 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/env"
-	"github.com/minio/pkg/wildcard"
-	"github.com/minio/pkg/workers"
+	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v2/wildcard"
+	"github.com/minio/pkg/v2/workers"
 )
 
 // expire: # Expire objects that match a condition

cmd/batch-handlers.go

@@ -1007,12 +1007,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// None of the provided metadata filters match skip the object.
 			return false
 		}
-		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
-		if (r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest {
-			return false
-		}
 
-		return true
+		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
+		return !((r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest)
 	}
 
 	u, err := url.Parse(r.Target.Endpoint)
@@ -1123,8 +1120,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
-		results := make(chan ObjectInfo, 100)
-		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, WalkOptions{
+		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, walkCh, WalkOptions{
 			Marker:   lastObject,
 			Filter:   selectObj,
 			AskDisks: walkQuorum,

cmd/bucket-lifecycle.go

@@ -219,24 +219,23 @@ type transitionState struct {
 	numWorkers int
 	killCh     chan struct{}
 
-	activeTasks int32
+	activeTasks          atomic.Int64
+	missedImmediateTasks atomic.Int64
 
 	lastDayMu    sync.RWMutex
 	lastDayStats map[string]*lastDayTierStats
 }
 
-func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc, blocking bool) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
 	task := transitionTask{objInfo: oi, event: event, src: src}
-	if blocking {
-		select {
-		case <-t.ctx.Done():
-		case t.transitionCh <- task:
-		}
-	} else {
-		select {
-		case <-t.ctx.Done():
-		case t.transitionCh <- task:
-		default:
+	select {
+	case <-t.ctx.Done():
+	case t.transitionCh <- task:
+	default:
+		switch src {
+		case lcEventSrc_s3PutObject, lcEventSrc_s3CopyObject, lcEventSrc_s3CompleteMultipartUpload:
+			// Update missed immediate tasks only for incoming requests.
+			t.missedImmediateTasks.Add(1)
 		}
 	}
 }
@@ -272,8 +271,14 @@ func (t *transitionState) PendingTasks() int {
 }
 
 // ActiveTasks returns the number of active (ongoing) ILM transition tasks.
-func (t *transitionState) ActiveTasks() int {
-	return int(atomic.LoadInt32(&t.activeTasks))
+func (t *transitionState) ActiveTasks() int64 {
+	return t.activeTasks.Load()
+}
+
+// MissedImmediateTasks returns the number of tasks - deferred to scanner due
+// to tasks channel being backlogged.
+func (t *transitionState) MissedImmediateTasks() int64 {
+	return t.missedImmediateTasks.Load()
 }
 
 // worker waits for transition tasks
@@ -288,7 +293,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			if !ok {
 				return
 			}
-			atomic.AddInt32(&t.activeTasks, 1)
+			t.activeTasks.Add(1)
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
 				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) {
 					if !strings.Contains(err.Error(), "use of closed network connection") {
@@ -306,7 +311,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				}
 				t.addLastDayStats(task.event.StorageClass, ts)
 			}
-			atomic.AddInt32(&t.activeTasks, -1)
+			t.activeTasks.Add(-1)
 		}
 	}
 }
@@ -379,7 +384,7 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
 		switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj, event, src, true)
+			globalTransitionState.queueTransitionTask(obj, event, src)
 		}
 	}
 }

cmd/bucket-replication-metrics.go

@@ -337,13 +337,13 @@ type SMA struct {
 	filledBuf bool
 }
 
-func newSMA(len int) *SMA {
-	if len <= 0 {
-		len = defaultWindowSize
+func newSMA(ln int) *SMA {
+	if ln <= 0 {
+		ln = defaultWindowSize
 	}
 	return &SMA{
-		buf:    make([]float64, len),
-		window: len,
+		buf:    make([]float64, ln),
+		window: ln,
 		idx:    0,
 	}
 }

cmd/bucket-replication.go

@@ -373,15 +373,14 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 			if oi.DeleteMarker && (validReplStatus || replicate) {
 				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
 				continue
-			} else {
-				// can be the case that other cluster is down and duplicate `mc rm --vid`
-				// is issued - this still needs to be replicated back to the other target
-				if !oi.VersionPurgeStatus.Empty() {
-					replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
-					dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
-				}
-				continue
 			}
+			// can be the case that other cluster is down and duplicate `mc rm --vid`
+			// is issued - this still needs to be replicated back to the other target
+			if !oi.VersionPurgeStatus.Empty() {
+				replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
+				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+			}
+			continue
 		}
 		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
 		// the target online status should not be used here while deciding
@@ -1469,6 +1468,24 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
 			},
 		}
+		if tagTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
+			ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
+			if err == nil {
+				dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
+			}
+		}
+		if retTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
+			ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
+			if err == nil {
+				dstOpts.Internal.RetentionTimestamp = ondiskTimestamp
+			}
+		}
+		if lholdTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok {
+			ondiskTimestamp, err := time.Parse(time.RFC3339, lholdTmStr)
+			if err == nil {
+				dstOpts.Internal.LegalholdTimestamp = ondiskTimestamp
+			}
+		}
 		if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil {
 			rinfo.ReplicationStatus = replication.Failed
 			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
@@ -1698,12 +1715,13 @@ type ReplicationPool struct {
 	activeWorkers    int32
 	activeMRFWorkers int32
 
-	objLayer ObjectLayer
-	ctx      context.Context
-	priority string
-	mu       sync.RWMutex
-	mrfMU    sync.Mutex
-	resyncer *replicationResyncer
+	objLayer   ObjectLayer
+	ctx        context.Context
+	priority   string
+	maxWorkers int
+	mu         sync.RWMutex
+	mrfMU      sync.Mutex
+	resyncer   *replicationResyncer
 
 	// workers:
 	workers    []chan ReplicationWorkerOperation
@@ -1749,9 +1767,13 @@ const (
 func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts) *ReplicationPool {
 	var workers, failedWorkers int
 	priority := "auto"
+	maxWorkers := WorkerMaxLimit
 	if opts.Priority != "" {
 		priority = opts.Priority
 	}
+	if opts.MaxWorkers > 0 {
+		maxWorkers = opts.MaxWorkers
+	}
 	switch priority {
 	case "fast":
 		workers = WorkerMaxLimit
@@ -1763,7 +1785,13 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		workers = WorkerAutoDefault
 		failedWorkers = MRFWorkerAutoDefault
 	}
+	if maxWorkers > 0 && workers > maxWorkers {
+		workers = maxWorkers
+	}
 
+	if maxWorkers > 0 && failedWorkers > maxWorkers {
+		failedWorkers = maxWorkers
+	}
 	pool := &ReplicationPool{
 		workers:         make([]chan ReplicationWorkerOperation, 0, workers),
 		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, LargeWorkerCount),
@@ -1775,6 +1803,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		ctx:             ctx,
 		objLayer:        o,
 		priority:        priority,
+		maxWorkers:      maxWorkers,
 	}
 
 	pool.AddLargeWorkers()
@@ -1930,7 +1959,7 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 }
 
 // ResizeWorkerPriority sets replication failed workers pool size
-func (p *ReplicationPool) ResizeWorkerPriority(pri string) {
+func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers int) {
 	var workers, mrfWorkers int
 	p.mu.Lock()
 	switch pri {
@@ -1950,7 +1979,15 @@ func (p *ReplicationPool) ResizeWorkerPriority(pri string) {
 			mrfWorkers = int(math.Min(float64(p.mrfWorkerSize+1), MRFWorkerAutoDefault))
 		}
 	}
+	if maxWorkers > 0 && workers > maxWorkers {
+		workers = maxWorkers
+	}
+
+	if maxWorkers > 0 && mrfWorkers > maxWorkers {
+		mrfWorkers = maxWorkers
+	}
 	p.priority = pri
+	p.maxWorkers = maxWorkers
 	p.mu.Unlock()
 	p.ResizeWorkers(workers, 0)
 	p.ResizeFailedWorkers(mrfWorkers)
@@ -2024,6 +2061,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
+		maxWorkers := p.maxWorkers
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
@@ -2031,16 +2069,18 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		case "slow":
 			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
 		default:
-			if p.ActiveWorkers() < WorkerMaxLimit {
+			maxWorkers = int(math.Min(float64(maxWorkers), WorkerMaxLimit))
+			if p.ActiveWorkers() < maxWorkers {
 				p.mu.RLock()
-				workers := int(math.Min(float64(len(p.workers)+1), WorkerMaxLimit))
+				workers := int(math.Min(float64(len(p.workers)+1), float64(maxWorkers)))
 				existing := len(p.workers)
 				p.mu.RUnlock()
 				p.ResizeWorkers(workers, existing)
 			}
-			if p.ActiveMRFWorkers() < MRFWorkerMaxLimit {
+			maxMRFWorkers := int(math.Min(float64(maxWorkers), MRFWorkerMaxLimit))
+			if p.ActiveMRFWorkers() < maxMRFWorkers {
 				p.mu.RLock()
-				workers := int(math.Min(float64(p.mrfWorkerSize+1), MRFWorkerMaxLimit))
+				workers := int(math.Min(float64(p.mrfWorkerSize+1), float64(maxMRFWorkers)))
 				p.mu.RUnlock()
 				p.ResizeFailedWorkers(workers)
 			}
@@ -2078,6 +2118,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		globalReplicationPool.queueMRFSave(doi.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
+		maxWorkers := p.maxWorkers
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
@@ -2085,9 +2126,10 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		case "slow":
 			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
 		default:
-			if p.ActiveWorkers() < WorkerMaxLimit {
+			maxWorkers = int(math.Min(float64(maxWorkers), WorkerMaxLimit))
+			if p.ActiveWorkers() < maxWorkers {
 				p.mu.RLock()
-				workers := int(math.Min(float64(len(p.workers)+1), WorkerMaxLimit))
+				workers := int(math.Min(float64(len(p.workers)+1), float64(maxWorkers)))
 				existing := len(p.workers)
 				p.mu.RUnlock()
 				p.ResizeWorkers(workers, existing)
@@ -2097,13 +2139,12 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 }
 
 type replicationPoolOpts struct {
-	Priority string
+	Priority   string
+	MaxWorkers int
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
-	globalReplicationPool = NewReplicationPool(ctx, objectAPI, replicationPoolOpts{
-		Priority: globalAPIConfig.getReplicationPriority(),
-	})
+	globalReplicationPool = NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts())
 	globalReplicationStats = NewReplicationStats(ctx, objectAPI)
 	go globalReplicationStats.trackEWMA()
 }

cmd/common-main.go

@@ -310,84 +310,123 @@ func checkUpdate(mode string) {
 	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }
 
-func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
-	var dir string
-	var dirSet bool
-
-	switch {
-	case ctx.IsSet(option):
-		dir = ctx.String(option)
-		dirSet = true
-	case ctx.GlobalIsSet(option):
-		dir = ctx.GlobalString(option)
-		dirSet = true
-		// cli package does not expose parent's option option.  Below code is workaround.
-		if dir == "" || dir == getDefaultDir() {
-			dirSet = false // Unset to false since GlobalIsSet() true is a false positive.
-			if ctx.Parent().GlobalIsSet(option) {
-				dir = ctx.Parent().GlobalString(option)
-				dirSet = true
-			}
-		}
-	default:
-		// Neither local nor global option is provided.  In this case, try to use
-		// default directory.
+func newConfigDir(dir string, dirSet bool, getDefaultDir func() string) (*ConfigDir, error) {
+	if dir == "" {
 		dir = getDefaultDir()
-		if dir == "" {
-			logger.FatalIf(errInvalidArgument, "%s option must be provided", option)
-		}
 	}
 
 	if dir == "" {
-		logger.FatalIf(errors.New("empty directory"), "%s directory cannot be empty", option)
+		if !dirSet {
+			return nil, fmt.Errorf("missing option must be provided")
+		}
+		return nil, fmt.Errorf("provided option cannot be empty")
 	}
 
 	// Disallow relative paths, figure out absolute paths.
 	dirAbs, err := filepath.Abs(dir)
-	logger.FatalIf(err, "Unable to fetch absolute path for %s=%s", option, dir)
+	if err != nil {
+		return nil, err
+	}
+	err = mkdirAllIgnorePerm(dirAbs)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create the directory `%s`: %w", dirAbs, err)
+	}
 
-	logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir)
-
-	return &ConfigDir{path: dirAbs}, dirSet
+	return &ConfigDir{path: dirAbs}, nil
 }
 
-func handleCommonCmdArgs(ctx *cli.Context) {
+func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	// Get "json" flag from command line argument and
-	// enable json and quite modes if json flag is turned on.
-	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
-	if globalCLIContext.JSON {
-		logger.EnableJSON()
-	}
-
+	ctxt.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
 	// Get quiet flag from command line argument.
-	globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
-	if globalCLIContext.Quiet {
-		logger.EnableQuiet()
-	}
-
+	ctxt.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
 	// Get anonymous flag from command line argument.
-	globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
-	if globalCLIContext.Anonymous {
-		logger.EnableAnonymous()
-	}
-
+	ctxt.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
 	// Fetch address option
-	addr := ctx.GlobalString("address")
-	if addr == "" || addr == ":"+GlobalMinioDefaultPort {
-		addr = ctx.String("address")
+	ctxt.Addr = ctx.GlobalString("address")
+	if ctxt.Addr == "" || ctxt.Addr == ":"+GlobalMinioDefaultPort {
+		ctxt.Addr = ctx.String("address")
 	}
 
 	// Fetch console address option
-	consoleAddr := ctx.GlobalString("console-address")
-	if consoleAddr == "" {
-		consoleAddr = ctx.String("console-address")
+	ctxt.ConsoleAddr = ctx.GlobalString("console-address")
+	if ctxt.ConsoleAddr == "" {
+		ctxt.ConsoleAddr = ctx.String("console-address")
 	}
 
+	// Check "no-compat" flag from command line argument.
+	ctxt.StrictS3Compat = true
+	if ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat") {
+		ctxt.StrictS3Compat = false
+	}
+
+	switch {
+	case ctx.IsSet("config-dir"):
+		ctxt.ConfigDir = ctx.String("config-dir")
+		ctxt.configDirSet = true
+	case ctx.GlobalIsSet("config-dir"):
+		ctxt.ConfigDir = ctx.GlobalString("config-dir")
+		ctxt.configDirSet = true
+	}
+
+	switch {
+	case ctx.IsSet("certs-dir"):
+		ctxt.CertsDir = ctx.String("certs-dir")
+		ctxt.certsDirSet = true
+	case ctx.GlobalIsSet("certs-dir"):
+		ctxt.CertsDir = ctx.GlobalString("certs-dir")
+		ctxt.certsDirSet = true
+	}
+
+	ctxt.FTP = ctx.StringSlice("ftp")
+	ctxt.SFTP = ctx.StringSlice("sftp")
+
+	ctxt.Interface = ctx.String("interface")
+	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
+	ctxt.ConnReadDeadline = ctx.Duration("conn-read-deadline")
+	ctxt.ConnWriteDeadline = ctx.Duration("conn-write-deadline")
+
+	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
+	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
+	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
+
+	if conf := ctx.String("config"); len(conf) > 0 {
+		err = mergeServerCtxtFromConfigFile(conf, ctxt)
+	} else {
+		err = mergeDisksLayoutFromArgs(serverCmdArgs(ctx), ctxt)
+	}
+
+	return err
+}
+
+func handleCommonArgs(ctxt serverCtxt) {
+	if ctxt.JSON {
+		logger.EnableJSON()
+	}
+	if ctxt.Quiet {
+		logger.EnableQuiet()
+	}
+	if ctxt.Anonymous {
+		logger.EnableAnonymous()
+	}
+
+	consoleAddr := ctxt.ConsoleAddr
+	addr := ctxt.Addr
+	configDir := ctxt.ConfigDir
+	configSet := ctxt.configDirSet
+	certsDir := ctxt.CertsDir
+	certsSet := ctxt.certsDirSet
+
 	if consoleAddr == "" {
 		p, err := xnet.GetFreePort()
 		if err != nil {
 			logger.FatalIf(err, "Unable to get free port for Console UI on the host")
 		}
+		// hold the port
+		l, err := net.Listen("TCP", fmt.Sprintf(":%s", p.String()))
+		if err == nil {
+			defer l.Close()
+		}
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
 
@@ -417,16 +456,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 
 	globalMinioAddr = addr
 
-	// Check "no-compat" flag from command line argument.
-	globalCLIContext.StrictS3Compat = true
-	if ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat") {
-		globalCLIContext.StrictS3Compat = false
-	}
-
 	// Set all config, certs and CAs directories.
-	var configSet, certsSet bool
-	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
-	globalCertsDir, certsSet = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
+	var err error
+	globalConfigDir, err = newConfigDir(configDir, configSet, defaultConfigDir.Get)
+	logger.FatalIf(err, "Unable to initialize the (deprecated) config directory")
+	globalCertsDir, err = newConfigDir(certsDir, certsSet, defaultCertsDir.Get)
+	logger.FatalIf(err, "Unable to initialize the certs directory")
 
 	// Remove this code when we deprecate and remove config-dir.
 	// This code is to make sure we inherit from the config-dir
@@ -438,9 +473,11 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
+}
 
-	// Check if we have configured a custom DNS cache TTL.
+func runDNSCache(ctx *cli.Context) {
 	dnsTTL := ctx.Duration("dns-cache-ttl")
+	// Check if we have configured a custom DNS cache TTL.
 	if dnsTTL <= 0 {
 		dnsTTL = 10 * time.Minute
 	}
@@ -621,7 +658,7 @@ func loadEnvVarsFromFiles() {
 	}
 }
 
-func handleCommonEnvVars() {
+func serverHandleEnvVars() {
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -749,6 +786,10 @@ func handleCommonEnvVars() {
 		}
 	}
 
+	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
+}
+
+func loadRootCredentials() {
 	// At this point, either both environment variables
 	// are defined or both are not defined.
 	// Check both cases and authenticate them if correctly defined
@@ -763,6 +804,9 @@ func handleCommonEnvVars() {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
 		hasCredentials = true
+	} else if globalServerCtxt.RootUser != "" && globalServerCtxt.RootPwd != "" {
+		user, password = globalServerCtxt.RootUser, globalServerCtxt.RootPwd
+		hasCredentials = true
 	}
 	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
@@ -782,8 +826,6 @@ func handleCommonEnvVars() {
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
-
-	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
 }
 
 // Initialize KMS global variable after valiadating and loading the configuration.

cmd/data-scanner.go

@@ -121,13 +121,13 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
 	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
 	if err != nil {
 		if !errors.Is(err, errConfigNotFound) {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
 		}
 		return backgroundHealInfo{}
 	}
 	var info backgroundHealInfo
 	if err = json.Unmarshal(buf, &info); err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
 	}
 	return info
 }
@@ -203,7 +203,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, "ns-scanner")
 			res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)}
 			if err != nil {
 				res["error"] = err.Error()
@@ -223,7 +223,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
 				tmp, _ = cycleInfo.MarshalMsg(tmp)
 				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
-				logger.LogIf(ctx, err)
+				logger.LogOnceIf(ctx, err, dataUsageBloomNamePath)
 			}
 		}
 	}
@@ -581,12 +581,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						foundAny = true
 						break
 					}
-					if next := f.updateCache.searchParent(parent); next == nil {
+					next := f.updateCache.searchParent(parent)
+					if next == nil {
 						foundAny = true
 						break
-					} else {
-						parent = *next
 					}
+					parent = *next
 				}
 				if !foundAny {
 					// Add non-compacted empty entry.
@@ -729,7 +729,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							versionID: "",
 						}, madmin.HealItemObject)
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, err)
+							logger.LogOnceIf(ctx, err, entry.name)
 						}
 						foundObjs = foundObjs || err == nil
 						return
@@ -746,7 +746,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						}, madmin.HealItemObject)
 						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, err)
+							logger.LogOnceIf(ctx, err, fiv.Name)
 						}
 						if err == nil {
 							successVersions++
@@ -1133,7 +1133,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 				err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
 				done()
 				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
+					logger.LogOnceIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
 				}
 			}
 		}
@@ -1190,7 +1190,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 	if obj.DeleteMarker {
 		return false
 	}
-	globalTransitionState.queueTransitionTask(obj, event, src, false)
+	globalTransitionState.queueTransitionTask(obj, event, src)
 	return true
 }
 
@@ -1199,7 +1199,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, obj.Name)
 		return false
 	}
 	// Notification already sent in *expireTransitionedObject*, just return 'true' here.

cmd/data-usage.go

@@ -56,7 +56,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			attempts = 1
 		}
 		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, dataUsageObjNamePath)
 		}
 		attempts++
 	}

cmd/endpoint-ellipses.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
+	"net/url"
 	"runtime"
 	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/pkg/v2/ellipses"
@@ -335,13 +338,128 @@ const (
 
 var globalCustomErasureDriveCount = false
 
-// CreateServerEndpoints - validates and creates new endpoints from input args, supports
-// both ellipses and without ellipses transparently.
-func createServerEndpoints(serverAddr string, args ...string) (
-	endpointServerPools EndpointServerPools, setupType SetupType, err error,
-) {
+type node struct {
+	nodeName string
+	disks    []string
+}
+
+type endpointsList []node
+
+func (el *endpointsList) add(arg string) error {
+	u, err := url.Parse(arg)
+	if err != nil {
+		return err
+	}
+	found := false
+	list := *el
+	for i := range list {
+		if list[i].nodeName == u.Host {
+			list[i].disks = append(list[i].disks, u.String())
+			found = true
+			break
+		}
+	}
+	if !found {
+		list = append(list, node{nodeName: u.Host, disks: []string{u.String()}})
+	}
+	*el = list
+	return nil
+}
+
+// buildDisksLayoutFromConfFile supports with and without ellipses transparently.
+func buildDisksLayoutFromConfFile(pools [][]string) (layout disksLayout, err error) {
+	if len(pools) == 0 {
+		return layout, errInvalidArgument
+	}
+
+	for _, list := range pools {
+		var endpointsList endpointsList
+
+		for _, arg := range list {
+			switch {
+			case ellipses.HasList(arg):
+				patterns, err := ellipses.FindListPatterns(arg)
+				if err != nil {
+					return layout, err
+				}
+				for _, exp := range patterns.Expand() {
+					for _, ep := range exp {
+						if err := endpointsList.add(ep); err != nil {
+							return layout, err
+						}
+					}
+				}
+			case ellipses.HasEllipses(arg):
+				patterns, err := ellipses.FindEllipsesPatterns(arg)
+				if err != nil {
+					return layout, err
+				}
+				for _, exp := range patterns.Expand() {
+					if err := endpointsList.add(strings.Join(exp, "")); err != nil {
+						return layout, err
+					}
+				}
+			default:
+				if err := endpointsList.add(arg); err != nil {
+					return layout, err
+				}
+			}
+		}
+
+		var stopping bool
+		var singleNode bool
+		var eps []string
+
+		for i := 0; ; i++ {
+			for _, node := range endpointsList {
+				if node.nodeName == "" {
+					singleNode = true
+				}
+
+				if len(node.disks) <= i {
+					stopping = true
+					continue
+				}
+				if stopping {
+					return layout, errors.New("number of disks per node does not match")
+				}
+				eps = append(eps, node.disks[i])
+			}
+			if stopping {
+				break
+			}
+		}
+
+		for _, node := range endpointsList {
+			if node.nodeName != "" && singleNode {
+				return layout, errors.New("all arguments must but either single node or distributed")
+			}
+		}
+
+		setArgs, err := GetAllSets(eps...)
+		if err != nil {
+			return layout, err
+		}
+
+		h := xxhash.New()
+		for _, s := range setArgs {
+			for _, d := range s {
+				h.WriteString(d)
+			}
+		}
+
+		layout.pools = append(layout.pools, poolDisksLayout{
+			cmdline: fmt.Sprintf("hash:%x", h.Sum(nil)),
+			layout:  setArgs,
+		})
+	}
+	return
+}
+
+// mergeDisksLayoutFromArgs supports with and without ellipses transparently.
+func mergeDisksLayoutFromArgs(args []string, ctxt *serverCtxt) (err error) {
 	if len(args) == 0 {
-		return nil, -1, errInvalidArgument
+		return errInvalidArgument
 	}
 
 	ok := true
@@ -349,44 +467,42 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		ok = ok && !ellipses.HasEllipses(arg)
 	}
 
+	var setArgs [][]string
+
 	// None of the args have ellipses use the old style.
 	if ok {
-		setArgs, err := GetAllSets(args...)
+		setArgs, err = GetAllSets(args...)
 		if err != nil {
-			return nil, -1, err
+			return err
 		}
-		endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
-		if err != nil {
-			return nil, -1, err
+		ctxt.Layout = disksLayout{
+			legacy: true,
+			pools:  []poolDisksLayout{{layout: setArgs}},
 		}
-		for i := range endpointList {
-			endpointList[i].SetPoolIndex(0)
-		}
-		endpointServerPools = append(endpointServerPools, PoolEndpoints{
-			Legacy:       true,
-			SetCount:     len(setArgs),
-			DrivesPerSet: len(setArgs[0]),
-			Endpoints:    endpointList,
-			CmdLine:      strings.Join(args, " "),
-			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
-		})
-		setupType = newSetupType
-		return endpointServerPools, setupType, nil
+		return
 	}
 
-	var poolArgs [][][]string
 	for _, arg := range args {
 		if !ellipses.HasEllipses(arg) && len(args) > 1 {
 			// TODO: support SNSD deployments to be decommissioned in future
-			return nil, -1, fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
+			return fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
 		}
-
-		setArgs, err := GetAllSets(arg)
+		setArgs, err = GetAllSets(arg)
 		if err != nil {
-			return nil, -1, err
+			return err
 		}
+		ctxt.Layout.pools = append(ctxt.Layout.pools, poolDisksLayout{cmdline: arg, layout: setArgs})
+	}
+	return
+}
 
-		poolArgs = append(poolArgs, setArgs)
+// CreateServerEndpoints - validates and creates new endpoints from input args, supports
+// both ellipses and without ellipses transparently.
+func createServerEndpoints(serverAddr string, poolArgs []poolDisksLayout, legacy bool) (
+	endpointServerPools EndpointServerPools, setupType SetupType, err error,
+) {
+	if len(poolArgs) == 0 {
+		return nil, -1, errInvalidArgument
 	}
 
 	poolEndpoints, setupType, err := CreatePoolEndpoints(serverAddr, poolArgs...)
@@ -396,11 +512,12 @@ func createServerEndpoints(serverAddr string, args ...string) (
 
 	for i, endpointList := range poolEndpoints {
 		if err = endpointServerPools.Add(PoolEndpoints{
-			SetCount:     len(poolArgs[i]),
-			DrivesPerSet: len(poolArgs[i][0]),
+			Legacy:       legacy,
+			SetCount:     len(poolArgs[i].layout),
+			DrivesPerSet: len(poolArgs[i].layout[0]),
 			Endpoints:    endpointList,
 			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
-			CmdLine:      args[i],
+			CmdLine:      poolArgs[i].cmdline,
 		}); err != nil {
 			return nil, -1, err
 		}

cmd/endpoint-ellipses_test.go

@@ -54,15 +54,20 @@ func TestCreateServerEndpoints(t *testing.T) {
 		{":9001", []string{"http://localhost:9001/export{01...64}"}, true},
 	}
 
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			_, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
+			srvCtxt := serverCtxt{}
+			err := mergeDisksLayoutFromArgs(testCase.args, &srvCtxt)
 			if err != nil && testCase.success {
-				t.Errorf("Expected success but failed instead %s", err)
+				t.Fatalf("Test %d: unexpected error: %v", i+1, err)
+			}
+			_, _, err = createServerEndpoints(testCase.serverAddr, srvCtxt.Layout.pools, srvCtxt.Layout.legacy)
+			if err != nil && testCase.success {
+				t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
 			}
 			if err == nil && !testCase.success {
-				t.Errorf("Expected failure but passed instead")
+				t.Errorf("Test %d: Expected failure but passed instead", i+1)
 			}
 		})
 	}

cmd/endpoint.go

@@ -803,11 +803,23 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 	return nil
 }
 
+func isEmptyLayout(poolsLayout ...poolDisksLayout) bool {
+	return len(poolsLayout) == 0 || len(poolsLayout[0].layout) == 0 || len(poolsLayout[0].layout[0]) == 0 || len(poolsLayout[0].layout[0][0]) == 0
+}
+
+func isSingleDriveLayout(poolsLayout ...poolDisksLayout) bool {
+	return len(poolsLayout) == 1 && len(poolsLayout[0].layout) == 1 && len(poolsLayout[0].layout[0]) == 1
+}
+
 // CreatePoolEndpoints creates a list of endpoints per pool, resolves their relevant hostnames and
 // discovers those are local or remote.
-func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints, SetupType, error) {
+func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]Endpoints, SetupType, error) {
 	var setupType SetupType
 
+	if isEmptyLayout(poolsLayout...) {
+		return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
+	}
+
 	// Check whether serverAddr is valid for this host.
 	if err := CheckLocalServerAddr(serverAddr); err != nil {
 		return nil, setupType, err
@@ -815,11 +827,11 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 
 	_, serverAddrPort := mustSplitHostPort(serverAddr)
 
-	poolEndpoints := make(PoolEndpointList, len(poolArgs))
+	poolEndpoints := make(PoolEndpointList, len(poolsLayout))
 
 	// For single arg, return single drive EC setup.
-	if len(poolArgs) == 1 && len(poolArgs[0]) == 1 && len(poolArgs[0][0]) == 1 && len(poolArgs[0][0][0]) == 1 {
-		endpoint, err := NewEndpoint(poolArgs[0][0][0])
+	if isSingleDriveLayout(poolsLayout...) {
+		endpoint, err := NewEndpoint(poolsLayout[0].layout[0][0])
 		if err != nil {
 			return nil, setupType, err
 		}
@@ -847,11 +859,11 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		return poolEndpoints, setupType, nil
 	}
 
-	for poolIdx, args := range poolArgs {
+	for poolIdx, pool := range poolsLayout {
 		var endpoints Endpoints
-		for setIdx, iargs := range args {
+		for setIdx, setLayout := range pool.layout {
 			// Convert args to endpoints
-			eps, err := NewEndpoints(iargs...)
+			eps, err := NewEndpoints(setLayout...)
 			if err != nil {
 				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 			}
@@ -1003,6 +1015,8 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		poolEndpoints[i] = endpoints
 	}
 
+	// TODO: ensure that each pool has at least two nodes in a distributed setup
+
 	publicIPs := env.Get(config.EnvPublicIPs, "")
 	if len(publicIPs) == 0 {
 		updateDomainIPs(uniqueArgs)
@@ -1021,199 +1035,6 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 	return poolEndpoints, setupType, nil
 }
 
-// CreateEndpoints - validates and creates new endpoints for given args.
-func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, error) {
-	var endpoints Endpoints
-	var setupType SetupType
-	var err error
-
-	// Check whether serverAddr is valid for this host.
-	if err = CheckLocalServerAddr(serverAddr); err != nil {
-		return endpoints, setupType, err
-	}
-
-	_, serverAddrPort := mustSplitHostPort(serverAddr)
-
-	// For single arg, return single drive setup.
-	if len(args) == 1 && len(args[0]) == 1 {
-		var endpoint Endpoint
-		endpoint, err = NewEndpoint(args[0][0])
-		if err != nil {
-			return endpoints, setupType, err
-		}
-		if err := endpoint.UpdateIsLocal(); err != nil {
-			return endpoints, setupType, err
-		}
-		if endpoint.Type() != PathEndpointType {
-			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
-		}
-
-		endpoint.SetPoolIndex(0)
-		endpoint.SetSetIndex(0)
-		endpoint.SetDiskIndex(0)
-
-		endpoints = append(endpoints, endpoint)
-		setupType = ErasureSDSetupType
-
-		// Check for cross device mounts if any.
-		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg(err.Error())
-		}
-
-		return endpoints, setupType, nil
-	}
-
-	for setIdx, iargs := range args {
-		// Convert args to endpoints
-		eps, err := NewEndpoints(iargs...)
-		if err != nil {
-			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
-		}
-
-		// Check for cross device mounts if any.
-		if err = checkCrossDeviceMounts(eps); err != nil {
-			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
-		}
-
-		for diskIdx := range eps {
-			eps[diskIdx].SetSetIndex(setIdx)
-			eps[diskIdx].SetDiskIndex(diskIdx)
-		}
-
-		endpoints = append(endpoints, eps...)
-	}
-
-	if len(endpoints) == 0 {
-		return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
-	}
-
-	// Return Erasure setup when all endpoints are path style.
-	if endpoints[0].Type() == PathEndpointType {
-		setupType = ErasureSetupType
-		return endpoints, setupType, nil
-	}
-
-	if err = endpoints.UpdateIsLocal(); err != nil {
-		return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
-	}
-
-	// Here all endpoints are URL style.
-	endpointPathSet := set.NewStringSet()
-	localEndpointCount := 0
-	localServerHostSet := set.NewStringSet()
-	localPortSet := set.NewStringSet()
-
-	for _, endpoint := range endpoints {
-		endpointPathSet.Add(endpoint.Path)
-		if endpoint.IsLocal {
-			localServerHostSet.Add(endpoint.Hostname())
-
-			var port string
-			_, port, err = net.SplitHostPort(endpoint.Host)
-			if err != nil {
-				port = serverAddrPort
-			}
-			localPortSet.Add(port)
-
-			localEndpointCount++
-		}
-	}
-
-	orchestrated := IsKubernetes() || IsDocker()
-	reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
-	// If not orchestrated
-	if !orchestrated &&
-		// and not setup in reverse proxy
-		!reverseProxy {
-		// Check whether same path is not used in endpoints of a host on different port.
-		// Only verify this on baremetal setups, DNS is not available in orchestrated
-		// environments so we can't do much here.
-		pathIPMap := make(map[string]set.StringSet)
-		hostIPCache := make(map[string]set.StringSet)
-		for _, endpoint := range endpoints {
-			host := endpoint.Hostname()
-			hostIPSet, ok := hostIPCache[host]
-			if !ok {
-				hostIPSet, err = getHostIP(host)
-				if err != nil {
-					return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
-				}
-				hostIPCache[host] = hostIPSet
-			}
-			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					return endpoints, setupType,
-						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
-				}
-				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-			} else {
-				pathIPMap[endpoint.Path] = hostIPSet
-			}
-		}
-	}
-
-	// Check whether same path is used for more than 1 local endpoints.
-	{
-		localPathSet := set.CreateStringSet()
-		for _, endpoint := range endpoints {
-			if !endpoint.IsLocal {
-				continue
-			}
-			if localPathSet.Contains(endpoint.Path) {
-				return endpoints, setupType,
-					config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
-			}
-			localPathSet.Add(endpoint.Path)
-		}
-	}
-
-	// Add missing port in all endpoints.
-	for i := range endpoints {
-		_, port, err := net.SplitHostPort(endpoints[i].Host)
-		if err != nil {
-			endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
-		} else if endpoints[i].IsLocal && serverAddrPort != port {
-			// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
-			endpoints[i].IsLocal = false
-		}
-	}
-
-	// All endpoints are pointing to local host
-	if len(endpoints) == localEndpointCount {
-		// If all endpoints have same port number, Just treat it as local erasure setup
-		// using URL style endpoints.
-		if len(localPortSet) == 1 {
-			if len(localServerHostSet) > 1 {
-				return endpoints, setupType,
-					config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
-			}
-			return endpoints, ErasureSetupType, nil
-		}
-
-		// Even though all endpoints are local, but those endpoints use different ports.
-		// This means it is DistErasure setup.
-	}
-
-	uniqueArgs := set.NewStringSet()
-	for _, endpoint := range endpoints {
-		uniqueArgs.Add(endpoint.Host)
-	}
-
-	// Error out if we have less than 2 unique servers.
-	if len(uniqueArgs.ToSlice()) < 2 && setupType == DistErasureSetupType {
-		err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)
-		return endpoints, setupType, err
-	}
-
-	publicIPs := env.Get(config.EnvPublicIPs, "")
-	if len(publicIPs) == 0 {
-		updateDomainIPs(uniqueArgs)
-	}
-
-	setupType = DistErasureSetupType
-	return endpoints, setupType, nil
-}
-
 // GetLocalPeer - returns local peer value, returns globalMinioAddr
 // for FS and Erasure mode. In case of distributed server return
 // the first element from the set of peers which indicate that

cmd/endpoint_test.go

@@ -225,24 +225,24 @@ func TestCreateEndpoints(t *testing.T) {
 
 	testCases := []struct {
 		serverAddr         string
-		args               [][]string
+		args               []string
 		expectedServerAddr string
 		expectedEndpoints  Endpoints
 		expectedSetupType  SetupType
 		expectedErr        error
 	}{
-		{"localhost", [][]string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")},
+		{"localhost", []string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")},
 
 		// Erasure Single Drive
-		{"localhost:9000", [][]string{{"http://localhost/d1"}}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
-		{":443", [][]string{{"/d1"}}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
-		{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
-		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
+		{"localhost:9000", []string{"http://localhost/d1"}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for SD setup")},
+		{":443", []string{"/d1"}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
+		{"localhost:10000", []string{"/d1"}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
+		{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 
 		// Erasure Setup with PathEndpointType
 		{
 			":1234",
-			[][]string{{"/d1", "/d2", "/d3", "/d4"}},
+			[]string{"/d1", "/d2", "/d3", "/d4"},
 			":1234",
 			Endpoints{
 				Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true},
@@ -253,49 +253,49 @@ func TestCreateEndpoints(t *testing.T) {
 			ErasureSetupType, nil,
 		},
 		// DistErasure Setup with URLEndpointType
-		{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
+		{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", Endpoints{
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d1"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d4"}, IsLocal: true},
-		}, ErasureSetupType, nil},
+		}, DistErasureSetupType, nil},
 		// DistErasure Setup with URLEndpointType having mixed naming to local host.
-		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
+		{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 
-		{":9001", [][]string{{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
+		{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
 
-		{":9000", [][]string{{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
+		{":9000", []string{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
 
 		// DistErasure type
-		{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
+		{"127.0.0.1:10000", []string{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", Endpoints{
 			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
 			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
 			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
 			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
+		{"127.0.0.1:10000", []string{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", Endpoints{
 			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
 			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
 			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
 			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", Endpoints{
+		{":80", []string{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", Endpoints{
 			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
 			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
 			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
 			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", Endpoints{
+		{":9000", []string{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", Endpoints{
 			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
 			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
 			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
 			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", Endpoints{
+		{":9000", []string{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}, ":9000", Endpoints{
 			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
 			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
 			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
@@ -303,7 +303,7 @@ func TestCreateEndpoints(t *testing.T) {
 		}, DistErasureSetupType, nil},
 
 		// DistErasure Setup using only local host.
-		{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", Endpoints{
+		{":9003", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}, ":9003", Endpoints{
 			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
 			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
 			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
@@ -311,24 +311,32 @@ func TestCreateEndpoints(t *testing.T) {
 		}, DistErasureSetupType, nil},
 	}
 
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
+		i := i
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
+			var srvCtxt serverCtxt
+			err := mergeDisksLayoutFromArgs(testCase.args, &srvCtxt)
+			if err != nil && testCase.expectedErr == nil {
+				t.Errorf("Test %d: unexpected error: %v", i+1, err)
+			}
+			pools, setupType, err := CreatePoolEndpoints(testCase.serverAddr, srvCtxt.Layout.pools...)
 			if err == nil && testCase.expectedErr != nil {
-				t.Errorf("error: expected = %v, got = <nil>", testCase.expectedErr)
+				t.Errorf("Test %d: expected = %v, got = <nil>", i+1, testCase.expectedErr)
 			}
 			if err == nil {
 				if setupType != testCase.expectedSetupType {
-					t.Errorf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+					t.Errorf("Test %d: setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
 				}
+				endpoints := pools[0]
 				if len(endpoints) != len(testCase.expectedEndpoints) {
-					t.Errorf("endpoints: expected = %d, got = %d", len(testCase.expectedEndpoints),
+					t.Errorf("Test %d: endpoints: expected = %d, got = %d", i+1, len(testCase.expectedEndpoints),
 						len(endpoints))
 				} else {
 					for i, endpoint := range endpoints {
 						if testCase.expectedEndpoints[i].String() != endpoint.String() {
-							t.Errorf("endpoints: expected = %s, got = %s",
+							t.Errorf("Test %d: endpoints: expected = %s, got = %s",
+								i+1,
 								testCase.expectedEndpoints[i],
 								endpoint)
 						}
@@ -336,7 +344,7 @@ func TestCreateEndpoints(t *testing.T) {
 				}
 			}
 			if err != nil && testCase.expectedErr == nil {
-				t.Errorf("error: expected = <nil>, got = %v, testCase: %v", err, testCase)
+				t.Errorf("Test %d: error: expected = <nil>, got = %v, testCase: %v", i+1, err, testCase)
 			}
 		})
 	}

cmd/erasure-healing.go

@@ -68,7 +68,7 @@ func (fi FileInfo) DataShardFixed() bool {
 	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
 }
 
-func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(string, metaCacheEntry) error) error {
+func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -101,7 +101,7 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 		minDisks:       1,
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
-			if err := healEntry(bucket, entry); err != nil {
+			if err := healEntry(bucket, entry, scanMode); err != nil {
 				cancel()
 			}
 		},
@@ -113,7 +113,7 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 				entry, _ = entries.firstFound()
 			}
 
-			if err := healEntry(bucket, *entry); err != nil {
+			if err := healEntry(bucket, *entry, scanMode); err != nil {
 				cancel()
 				return
 			}

cmd/erasure-healing_test.go

@@ -647,8 +647,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
-		func(bucket, object, vid string) error {
-			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
 		}); err != nil {
 		t.Fatal(err)
@@ -694,8 +694,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
-		func(bucket, object, vid string) error {
-			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
 		}); err != nil {
 		t.Fatal(err)
@@ -743,8 +743,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
-		func(bucket, object, vid string) error {
-			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
 		}); err != nil {
 		t.Fatal(err)

cmd/erasure-multipart.go

@@ -1290,13 +1290,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		})
 	}
 
-	// Check if there is any offline disk and add it to the MRF list
-	for _, disk := range onlineDisks {
-		if disk != nil && disk.IsOnline() {
-			continue
+	if !opts.Speedtest && !versionsDisparity {
+		// Check if there is any offline disk and add it to the MRF list
+		for _, disk := range onlineDisks {
+			if disk != nil && disk.IsOnline() {
+				continue
+			}
+			er.addPartial(bucket, object, fi.VersionID)
+			break
 		}
-		er.addPartial(bucket, object, fi.VersionID)
-		break
 	}
 
 	for i := 0; i < len(onlineDisks); i++ {

cmd/erasure-object.go

@@ -99,7 +99,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(srcBucket, minioMetaBucket) {
-			_, derr := er.deleteIfDangling(ctx, srcBucket, srcObject, metaArr, errs, nil, srcOpts)
+			_, derr := er.deleteIfDangling(context.Background(), srcBucket, srcObject, metaArr, errs, nil, srcOpts)
 			if derr != nil {
 				err = derr
 			}
@@ -413,6 +413,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 							queued:    time.Now(),
 							setIndex:  er.setIndex,
 							poolIndex: er.poolIndex,
+							scanMode:  scan,
 						})
 					})
 					// Healing is triggered and we have written
@@ -910,7 +911,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 
 	if err != nil {
 		if shouldCheckForDangling(err, errs, bucket) {
-			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
 			}
@@ -1036,6 +1037,23 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	var versionsDisparity bool
 
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if err != nil {
+		dg := errgroup.WithNErrs(len(disks))
+		for index, nerr := range errs {
+			if nerr != nil {
+				continue
+			}
+			index := index
+			// When we are going to return error, attempt to delete success
+			// on some of the drives, if we cannot we do not have to notify
+			// caller this dangling object will be now scheduled to be removed
+			// via active healing.
+			dg.Go(func() error {
+				return disks[index].DeleteVersion(context.Background(), dstBucket, dstEntry, metadata[index], false)
+			}, index)
+		}
+		dg.Wait()
+	}
 	if err == nil {
 		versions := reduceCommonVersions(diskVersions, writeQuorum)
 		for index, dversions := range diskVersions {
@@ -1200,7 +1218,7 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 }
 
 // Heal up to two versions of one object when there is disparity between disks
-func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
+func healObjectVersionsDisparity(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
 	if entry.isDir() {
 		return nil
 	}
@@ -1224,13 +1242,13 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
 
 	fivs, err := entry.fileInfoVersions(bucket)
 	if err != nil {
-		go healObject(bucket, entry.name, "", madmin.HealDeepScan)
+		healObject(bucket, entry.name, "", madmin.HealDeepScan)
 		return err
 	}
 
 	if len(fivs.Versions) <= 2 {
 		for _, version := range fivs.Versions {
-			go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+			healObject(bucket, entry.name, version.VersionID, scanMode)
 		}
 	}
 
@@ -1531,18 +1549,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// For speedtest objects do not attempt to heal them.
 	if !opts.Speedtest {
-		// Whether a disk was initially or becomes offline
-		// during this upload, send it to the MRF list.
-		for i := 0; i < len(onlineDisks); i++ {
-			if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
-				continue
+		// When there is versions disparity we are healing
+		// the content implicitly for all versions, we can
+		// avoid triggering another MRF heal for offline drives.
+		if !versionsDisparity {
+			// Whether a disk was initially or becomes offline
+			// during this upload, send it to the MRF list.
+			for i := 0; i < len(onlineDisks); i++ {
+				if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+					continue
+				}
+
+				er.addPartial(bucket, object, fi.VersionID)
+				break
 			}
-
-			er.addPartial(bucket, object, fi.VersionID)
-			break
-		}
-
-		if versionsDisparity {
+		} else {
 			globalMRFState.addPartialOp(partialOperation{
 				bucket:      bucket,
 				object:      object,
@@ -2028,7 +2049,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
 			}
@@ -2101,7 +2122,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
 			}

cmd/erasure-server-pool.go

@@ -2108,10 +2108,10 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 }
 
 // HealObjectFn closure function heals the object.
-type HealObjectFn func(bucket, object, versionID string) error
+type HealObjectFn func(bucket, object, versionID string, scanMode madmin.HealScanMode) error
 
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
-	healEntry := func(bucket string, entry metaCacheEntry) error {
+	healEntry := func(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
 		if entry.isDir() {
 			return nil
 		}
@@ -2134,7 +2134,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		}
 		fivs, err := entry.fileInfoVersions(bucket)
 		if err != nil {
-			return healObjectFn(bucket, entry.name, "")
+			return healObjectFn(bucket, entry.name, "", scanMode)
 		}
 		if opts.Remove && !opts.DryRun {
 			err := z.CheckAbandonedParts(ctx, bucket, entry.name, opts)
@@ -2143,7 +2143,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			}
 		}
 		for _, version := range fivs.Versions {
-			err := healObjectFn(bucket, version.Name, version.VersionID)
+			err := healObjectFn(bucket, version.Name, version.VersionID, scanMode)
 			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 				return err
 			}
@@ -2167,7 +2167,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
 
-				errs[idx] = set.listAndHeal(bucket, prefix, healEntry)
+				errs[idx] = set.listAndHeal(bucket, prefix, opts.ScanMode, healEntry)
 			}(idx, set)
 		}
 		wg.Wait()

cmd/erasure-utils.go

@@ -72,15 +72,15 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 			// Decrement offset.
 			offset -= int64(len(block))
 			continue
-		} else {
-			// Skip until offset.
-			block = block[offset:]
-
-			// Reset the offset for next iteration to read everything
-			// from subsequent blocks.
-			offset = 0
 		}
 
+		// Skip until offset.
+		block = block[offset:]
+
+		// Reset the offset for next iteration to read everything
+		// from subsequent blocks.
+		offset = 0
+
 		// We have written all the blocks, write the last remaining block.
 		if write < int64(len(block)) {
 			n, err := dst.Write(block[:write])

cmd/event-notification.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"runtime"
 	"strings"
 	"sync"
 
@@ -36,16 +37,14 @@ import (
 type EventNotifier struct {
 	sync.RWMutex
 	targetList     *event.TargetList
-	targetResCh    chan event.TargetIDResult
 	bucketRulesMap map[string]event.RulesMap
 }
 
 // NewEventNotifier - creates new event notification object.
-func NewEventNotifier() *EventNotifier {
+func NewEventNotifier(ctx context.Context) *EventNotifier {
 	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.InitBucketTargets()
 	return &EventNotifier{
-		targetList:     event.NewTargetList(),
-		targetResCh:    make(chan event.TargetIDResult),
+		targetList:     event.NewTargetList(ctx),
 		bucketRulesMap: make(map[string]event.RulesMap),
 	}
 }
@@ -90,6 +89,11 @@ func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
 	evnot.AddRulesMap(bucket.Name, config.ToRulesMap())
 }
 
+// Targets returns all the registered targets
+func (evnot *EventNotifier) Targets() []event.Target {
+	return evnot.targetList.Targets()
+}
+
 // InitBucketTargets - initializes event notification system from notification.xml of all buckets.
 func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI ObjectLayer) error {
 	if objAPI == nil {
@@ -99,17 +103,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 	if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil {
 		return err
 	}
-
-	go func() {
-		for res := range evnot.targetResCh {
-			if res.Err != nil {
-				reqInfo := &logger.ReqInfo{}
-				reqInfo.AppendTags("targetID", res.ID.String())
-				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
-			}
-		}
-	}()
-
+	evnot.targetList = evnot.targetList.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4)
 	return nil
 }
 
@@ -159,7 +153,7 @@ func (evnot *EventNotifier) Send(args eventArgs) {
 	}
 
 	// If MINIO_API_SYNC_EVENTS is set, send events synchronously.
-	evnot.targetList.Send(args.ToEvent(true), targetIDSet, evnot.targetResCh, globalAPIConfig.isSyncEventsEnabled())
+	evnot.targetList.Send(args.ToEvent(true), targetIDSet, globalAPIConfig.isSyncEventsEnabled())
 }
 
 type eventArgs struct {

cmd/ftp-server-driver.go

@@ -248,12 +248,19 @@ func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string)
 	defer stopFn(err)
 
 	if globalIAMSys.LDAPConfig.Enabled() {
-		ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(username, password)
-		if err != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), username)
+		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
 			return false, err
 		}
-		ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, groupDistNames...)
-		return len(ldapPolicies) > 0, nil
+		if errors.Is(err, errNoSuchServiceAccount) {
+			ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(username, password)
+			if err != nil {
+				return false, err
+			}
+			ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, groupDistNames...)
+			return len(ldapPolicies) > 0, nil
+		}
+		return subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), []byte(password)) == 1, nil
 	}
 
 	ui, ok := globalIAMSys.GetUser(context.Background(), username)
@@ -269,58 +276,70 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 		return nil, errNoSuchUser
 	}
 	if !ok && globalIAMSys.LDAPConfig.Enabled() {
-		targetUser, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ctx.Sess.LoginUser())
-		if err != nil {
-			return nil, err
-		}
-		ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
-		if len(ldapPolicies) == 0 {
-			return nil, errAuthentication
-		}
-		expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
-		if err != nil {
-			return nil, err
-		}
-		claims := make(map[string]interface{})
-		claims[expClaim] = UTCNow().Add(expiryDur).Unix()
-		claims[ldapUser] = targetUser
-		claims[ldapUserN] = ctx.Sess.LoginUser()
-
-		cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
-		if err != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), ctx.Sess.LoginUser())
+		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
 			return nil, err
 		}
 
-		// Set the parent of the temporary access key, this is useful
-		// in obtaining service accounts by this cred.
-		cred.ParentUser = targetUser
+		var mcreds *credentials.Credentials
+		if errors.Is(err, errNoSuchServiceAccount) {
+			targetUser, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ctx.Sess.LoginUser())
+			if err != nil {
+				return nil, err
+			}
+			ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
+			if len(ldapPolicies) == 0 {
+				return nil, errAuthentication
+			}
+			expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
+			if err != nil {
+				return nil, err
+			}
+			claims := make(map[string]interface{})
+			claims[expClaim] = UTCNow().Add(expiryDur).Unix()
+			claims[ldapUser] = targetUser
+			claims[ldapUserN] = ctx.Sess.LoginUser()
 
-		// Set this value to LDAP groups, LDAP user can be part
-		// of large number of groups
-		cred.Groups = targetGroups
+			cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
+			if err != nil {
+				return nil, err
+			}
 
-		// Set the newly generated credentials, policyName is empty on purpose
-		// LDAP policies are applied automatically using their ldapUser, ldapGroups
-		// mapping.
-		updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
-		if err != nil {
-			return nil, err
+			// Set the parent of the temporary access key, this is useful
+			// in obtaining service accounts by this cred.
+			cred.ParentUser = targetUser
+
+			// Set this value to LDAP groups, LDAP user can be part
+			// of large number of groups
+			cred.Groups = targetGroups
+
+			// Set the newly generated credentials, policyName is empty on purpose
+			// LDAP policies are applied automatically using their ldapUser, ldapGroups
+			// mapping.
+			updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
+			if err != nil {
+				return nil, err
+			}
+
+			// Call hook for site replication.
+			logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+				Type: madmin.SRIAMItemSTSAcc,
+				STSCredential: &madmin.SRSTSCredential{
+					AccessKey:    cred.AccessKey,
+					SecretKey:    cred.SecretKey,
+					SessionToken: cred.SessionToken,
+					ParentUser:   cred.ParentUser,
+				},
+				UpdatedAt: updatedAt,
+			}))
+
+			mcreds = credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken)
+		} else {
+			mcreds = credentials.NewStaticV4(sa.Credentials.AccessKey, sa.Credentials.SecretKey, "")
 		}
 
-		// Call hook for site replication.
-		logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
-			Type: madmin.SRIAMItemSTSAcc,
-			STSCredential: &madmin.SRSTSCredential{
-				AccessKey:    cred.AccessKey,
-				SecretKey:    cred.SecretKey,
-				SessionToken: cred.SessionToken,
-				ParentUser:   cred.ParentUser,
-			},
-			UpdatedAt: updatedAt,
-		}))
-
 		return minio.New(driver.endpoint, &minio.Options{
-			Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+			Creds:     mcreds,
 			Secure:    globalIsTLS,
 			Transport: globalRemoteFTPClientTransport,
 		})

cmd/ftp-server.go

@@ -23,7 +23,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/cli"
 	"github.com/minio/minio/internal/logger"
 	ftp "goftp.io/server/v2"
 )
@@ -69,9 +68,7 @@ func (log *minioLogger) PrintResponse(sessionID string, code int, message string
 	}
 }
 
-func startFTPServer(c *cli.Context) {
-	args := c.StringSlice("ftp")
-
+func startFTPServer(args []string) {
 	var (
 		port          int
 		publicIP      string

cmd/generic-handlers.go

@@ -153,7 +153,7 @@ func setBrowserRedirectMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		read := r.Method == http.MethodGet || r.Method == http.MethodHead
 		// Re-direction is handled specifically for browser requests.
-		if guessIsBrowserReq(r) && read && globalBrowserRedirect {
+		if !guessIsHealthCheckReq(r) && guessIsBrowserReq(r) && read && globalBrowserRedirect {
 			// Fetch the redirect location if any.
 			if u := getRedirectLocation(r); u != nil {
 				// Employ a temporary re-direct.

cmd/globals.go

@@ -127,13 +127,46 @@ const (
 	tlsClientSessionCacheSize = 100
 )
 
-var globalCLIContext = struct {
-	JSON, Quiet    bool
-	Anonymous      bool
-	StrictS3Compat bool
-}{}
+type poolDisksLayout struct {
+	cmdline string
+	layout  [][]string
+}
+
+type disksLayout struct {
+	legacy bool
+	pools  []poolDisksLayout
+}
+
+type serverCtxt struct {
+	JSON, Quiet               bool
+	Anonymous                 bool
+	StrictS3Compat            bool
+	Addr, ConsoleAddr         string
+	ConfigDir, CertsDir       string
+	configDirSet, certsDirSet bool
+	Interface                 string
+
+	RootUser, RootPwd string
+
+	FTP  []string
+	SFTP []string
+
+	UserTimeout       time.Duration
+	ConnReadDeadline  time.Duration
+	ConnWriteDeadline time.Duration
+
+	ShutdownTimeout   time.Duration
+	IdleTimeout       time.Duration
+	ReadHeaderTimeout time.Duration
+
+	// The layout of disks as interpreted
+	Layout disksLayout
+}
 
 var (
+	// Global user opts context
+	globalServerCtxt serverCtxt
+
 	// Indicates if the running minio server is distributed setup.
 	globalIsDistErasure = false
 

cmd/handler-api.go

@@ -42,9 +42,10 @@ type apiConfig struct {
 	listQuorum       string
 	corsAllowOrigins []string
 	// total drives per erasure set across pools.
-	totalDriveCount     int
-	replicationPriority string
-	transitionWorkers   int
+	totalDriveCount       int
+	replicationPriority   string
+	replicationMaxWorkers int
+	transitionWorkers     int
 
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
@@ -152,10 +153,11 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	}
 	t.listQuorum = listQuorum
 	if globalReplicationPool != nil &&
-		cfg.ReplicationPriority != t.replicationPriority {
-		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority)
+		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers) {
+		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers)
 	}
 	t.replicationPriority = cfg.ReplicationPriority
+	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
 	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
 		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
 	}
@@ -334,15 +336,21 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
-func (t *apiConfig) getReplicationPriority() string {
+func (t *apiConfig) getReplicationOpts() replicationPoolOpts {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
 	if t.replicationPriority == "" {
-		return "auto"
+		return replicationPoolOpts{
+			Priority:   "auto",
+			MaxWorkers: WorkerMaxLimit,
+		}
 	}
 
-	return t.replicationPriority
+	return replicationPoolOpts{
+		Priority:   t.replicationPriority,
+		MaxWorkers: t.replicationMaxWorkers,
+	}
 }
 
 func (t *apiConfig) getTransitionWorkers() int {

cmd/metrics-realtime.go

@@ -91,7 +91,7 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		}
 		cm, err := c.Times(false)
 		if err != nil {
-			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v  (cputimes)", globalMinioAddr, err.Error()))
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuTimes)", globalMinioAddr, err.Error()))
 		} else {
 			// not collecting per-cpu stats, so there will be only one element
 			if len(cm) == 1 {
@@ -100,6 +100,13 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 				m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", globalMinioAddr, len(cm)))
 			}
 		}
+		cpuCount, err := c.Counts(true)
+		if err != nil {
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuCount)", globalMinioAddr, err.Error()))
+		} else {
+			m.Aggregated.CPU.CPUCount = cpuCount
+		}
+
 		loadStat, err := load.Avg()
 		if err != nil {
 			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", globalMinioAddr, err.Error()))

cmd/metrics-resource.go

@@ -53,6 +53,7 @@ const (
 
 	// memory stats
 	memUsed      MetricName = "used"
+	memUsedPerc  MetricName = "used_perc"
 	memFree      MetricName = "free"
 	memShared    MetricName = "shared"
 	memBuffers   MetricName = "buffers"
@@ -60,15 +61,18 @@ const (
 	memAvailable MetricName = "available"
 
 	// cpu stats
-	cpuUser   MetricName = "user"
-	cpuSystem MetricName = "system"
-	cpuIOWait MetricName = "iowait"
-	cpuIdle   MetricName = "idle"
-	cpuNice   MetricName = "nice"
-	cpuSteal  MetricName = "steal"
-	cpuLoad1  MetricName = "load1"
-	cpuLoad5  MetricName = "load5"
-	cpuLoad15 MetricName = "load15"
+	cpuUser       MetricName = "user"
+	cpuSystem     MetricName = "system"
+	cpuIOWait     MetricName = "iowait"
+	cpuIdle       MetricName = "idle"
+	cpuNice       MetricName = "nice"
+	cpuSteal      MetricName = "steal"
+	cpuLoad1      MetricName = "load1"
+	cpuLoad5      MetricName = "load5"
+	cpuLoad15     MetricName = "load15"
+	cpuLoad1Perc  MetricName = "load1_perc"
+	cpuLoad5Perc  MetricName = "load5_perc"
+	cpuLoad15Perc MetricName = "load15_perc"
 )
 
 var (
@@ -126,6 +130,7 @@ func init() {
 		interfaceTxErrors: "Transmit errors in " + interval,
 		total:             "Total memory on the node",
 		memUsed:           "Used memory on the node",
+		memUsedPerc:       "Used memory percentage on the node",
 		memFree:           "Free memory on the node",
 		memShared:         "Shared memory on the node",
 		memBuffers:        "Buffers memory on the node",
@@ -151,6 +156,9 @@ func init() {
 		cpuLoad1:          "CPU load average 1min",
 		cpuLoad5:          "CPU load average 5min",
 		cpuLoad15:         "CPU load average 15min",
+		cpuLoad1Perc:      "CPU load average 1min (perentage)",
+		cpuLoad5Perc:      "CPU load average 5min (percentage)",
+		cpuLoad15Perc:     "CPU load average 15min (percentage)",
 	}
 	resourceMetricsGroups = []*MetricsGroup{
 		getResourceMetrics(),
@@ -283,6 +291,8 @@ func collectLocalResourceMetrics() {
 				stats := hm.Mem.Info
 				updateResourceMetrics(memSubsystem, total, float64(stats.Total), labels, false)
 				updateResourceMetrics(memSubsystem, memUsed, float64(stats.Used), labels, false)
+				perc := math.Round(float64(stats.Used*100*100)/float64(stats.Total)) / 100
+				updateResourceMetrics(memSubsystem, memUsedPerc, perc, labels, false)
 				updateResourceMetrics(memSubsystem, memFree, float64(stats.Free), labels, false)
 				updateResourceMetrics(memSubsystem, memShared, float64(stats.Shared), labels, false)
 				updateResourceMetrics(memSubsystem, memBuffers, float64(stats.Buffers), labels, false)
@@ -312,6 +322,14 @@ func collectLocalResourceMetrics() {
 					updateResourceMetrics(cpuSubsystem, cpuLoad1, ls.Load1, labels, false)
 					updateResourceMetrics(cpuSubsystem, cpuLoad5, ls.Load5, labels, false)
 					updateResourceMetrics(cpuSubsystem, cpuLoad15, ls.Load15, labels, false)
+					if hm.CPU.CPUCount > 0 {
+						perc := math.Round(ls.Load1*100*100/float64(hm.CPU.CPUCount)) / 100
+						updateResourceMetrics(cpuSubsystem, cpuLoad1Perc, perc, labels, false)
+						perc = math.Round(ls.Load5*100*100/float64(hm.CPU.CPUCount)) / 100
+						updateResourceMetrics(cpuSubsystem, cpuLoad5Perc, perc, labels, false)
+						perc = math.Round(ls.Load15*100*100/float64(hm.CPU.CPUCount)) / 100
+						updateResourceMetrics(cpuSubsystem, cpuLoad15Perc, perc, labels, false)
+					}
 				}
 			}
 			break // only one host expected

cmd/metrics-v2.go

@@ -256,6 +256,7 @@ const (
 	expiryPendingTasks     MetricName = "expiry_pending_tasks"
 	transitionPendingTasks MetricName = "transition_pending_tasks"
 	transitionActiveTasks  MetricName = "transition_active_tasks"
+	transitionMissedTasks  MetricName = "transition_missed_immediate_tasks"
 
 	transitionedBytes    MetricName = "transitioned_bytes"
 	transitionedObjects  MetricName = "transitioned_objects"
@@ -1707,6 +1708,16 @@ func getTransitionActiveTasksMD() MetricDescription {
 	}
 }
 
+func getTransitionMissedTasksMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: ilmSubsystem,
+		Name:      transitionMissedTasks,
+		Help:      "Number of missed immediate ILM transition tasks",
+		Type:      gaugeMetric,
+	}
+}
+
 func getExpiryPendingTasksMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -1781,17 +1792,22 @@ func getILMNodeMetrics() *MetricsGroup {
 		trActiveTasks := Metric{
 			Description: getTransitionActiveTasksMD(),
 		}
+		trMissedTasks := Metric{
+			Description: getTransitionMissedTasksMD(),
+		}
 		if globalExpiryState != nil {
 			expPendingTasks.Value = float64(globalExpiryState.PendingTasks())
 		}
 		if globalTransitionState != nil {
 			trPendingTasks.Value = float64(globalTransitionState.PendingTasks())
 			trActiveTasks.Value = float64(globalTransitionState.ActiveTasks())
+			trMissedTasks.Value = float64(globalTransitionState.MissedImmediateTasks())
 		}
 		return []Metric{
 			expPendingTasks,
 			trPendingTasks,
 			trActiveTasks,
+			trMissedTasks,
 		}
 	})
 	return mg
@@ -2459,6 +2475,16 @@ func getNotificationMetrics() *MetricsGroup {
 				},
 				Value: float64(nstats.EventsSkipped),
 			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: notifySubsystem,
+					Name:      "events_errors_total",
+					Help:      "Events that were failed while sending to target",
+					Type:      counterMetric,
+				},
+				Value: float64(nstats.EventsErrorsTotal),
+			})
 			metrics = append(metrics, Metric{
 				Description: MetricDescription{
 					Namespace: minioNamespace,

cmd/mrf.go

@@ -38,6 +38,7 @@ type partialOperation struct {
 	allVersions         bool
 	setIndex, poolIndex int
 	queued              time.Time
+	scanMode            madmin.HealScanMode
 }
 
 // mrfState sncapsulates all the information
@@ -97,19 +98,23 @@ func (m *mrfState) healRoutine() {
 				// let recently failed networks to reconnect
 				// making MRF wait for 1s before retrying,
 				// i.e 4 reconnect attempts.
-				time.Sleep(1 * time.Second)
+				time.Sleep(time.Second)
 			}
 
 			// wait on timer per heal
 			wait := healSleeper.Timer(context.Background())
 
+			scan := madmin.HealNormalScan
+			if u.scanMode != 0 {
+				scan = u.scanMode
+			}
 			if u.object == "" {
-				healBucket(u.bucket, madmin.HealNormalScan)
+				healBucket(u.bucket, scan)
 			} else {
 				if u.allVersions {
-					m.pools.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, healObjectVersionsDisparity)
+					m.pools.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, u.scanMode, healObjectVersionsDisparity)
 				} else {
-					healObject(u.bucket, u.object, u.versionID, madmin.HealNormalScan)
+					healObject(u.bucket, u.object, u.versionID, scan)
 				}
 			}
 

cmd/notification.go

@@ -975,8 +975,11 @@ func getOfflineDisks(offlineHost string, endpoints EndpointServerPools) []madmin
 		for _, ep := range pool.Endpoints {
 			if offlineHost == "" && ep.IsLocal || offlineHost == ep.Host {
 				offlineDisks = append(offlineDisks, madmin.Disk{
-					Endpoint: ep.String(),
-					State:    string(madmin.ItemOffline),
+					Endpoint:  ep.String(),
+					State:     string(madmin.ItemOffline),
+					PoolIndex: ep.PoolIdx,
+					SetIndex:  ep.SetIdx,
+					DiskIndex: ep.DiskIdx,
 				})
 			}
 		}

cmd/server-main.go

@@ -51,10 +51,16 @@ import (
 	"github.com/minio/pkg/v2/certs"
 	"github.com/minio/pkg/v2/env"
 	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
 )
 
 // ServerFlags - server command specific flags
 var ServerFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:   "config",
+		Usage:  "specify server configuration via YAML configuration",
+		EnvVar: "MINIO_CONFIG",
+	},
 	cli.StringFlag{
 		Name:   "address",
 		Value:  ":" + GlobalMinioDefaultPort,
@@ -226,9 +232,57 @@ func serverCmdArgs(ctx *cli.Context) []string {
 	return strings.Fields(v)
 }
 
-func serverHandleCmdArgs(ctx *cli.Context) {
-	// Handle common command args.
-	handleCommonCmdArgs(ctx)
+func mergeServerCtxtFromConfigFile(configFile string, ctxt *serverCtxt) error {
+	rd, err := Open(configFile)
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
+	cf := &config.ServerConfig{}
+	dec := yaml.NewDecoder(rd)
+	dec.SetStrict(true)
+	if err = dec.Decode(cf); err != nil {
+		return err
+	}
+	if cf.Version != "v1" {
+		return fmt.Errorf("unexpected version: %s", cf.Version)
+	}
+
+	ctxt.RootUser = cf.RootUser
+	ctxt.RootPwd = cf.RootPwd
+
+	if cf.Addr != "" {
+		ctxt.Addr = cf.Addr
+	}
+	if cf.ConsoleAddr != "" {
+		ctxt.ConsoleAddr = cf.ConsoleAddr
+	}
+	if cf.CertsDir != "" {
+		ctxt.CertsDir = cf.CertsDir
+		ctxt.certsDirSet = true
+	}
+
+	if cf.Options.FTP.Address != "" {
+		ctxt.FTP = append(ctxt.FTP, fmt.Sprintf("address=%s", cf.Options.FTP.Address))
+	}
+	if cf.Options.FTP.PassivePortRange != "" {
+		ctxt.FTP = append(ctxt.FTP, fmt.Sprintf("passive-port-range=%s", cf.Options.FTP.PassivePortRange))
+	}
+
+	if cf.Options.SFTP.Address != "" {
+		ctxt.SFTP = append(ctxt.SFTP, fmt.Sprintf("address=%s", cf.Options.SFTP.Address))
+	}
+	if cf.Options.SFTP.SSHPrivateKey != "" {
+		ctxt.SFTP = append(ctxt.SFTP, fmt.Sprintf("ssh-private-key=%s", cf.Options.SFTP.SSHPrivateKey))
+	}
+
+	ctxt.Layout, err = buildDisksLayoutFromConfFile(cf.Pools)
+	return err
+}
+
+func serverHandleCmdArgs(ctxt serverCtxt) {
+	handleCommonArgs(ctxt)
 
 	logger.FatalIf(CheckLocalServerAddr(globalMinioAddr), "Unable to validate passed arguments")
 
@@ -251,7 +305,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	// Register root CAs for remote ENVs
 	env.RegisterGlobalCAs(globalRootCAs)
 
-	globalEndpoints, setupType, err = createServerEndpoints(globalMinioAddr, serverCmdArgs(ctx)...)
+	globalEndpoints, setupType, err = createServerEndpoints(globalMinioAddr, ctxt.Layout.pools, ctxt.Layout.legacy)
 	logger.FatalIf(err, "Invalid command line arguments")
 	globalNodes = globalEndpoints.GetNodes()
 
@@ -262,7 +316,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	}
 	globalIsErasureSD = (setupType == ErasureSDSetupType)
 	if globalDynamicAPIPort && globalIsDistErasure {
-		logger.FatalIf(errInvalidArgument, "Invalid --address=\"%s\", port '0' is not allowed in a distributed erasure coded setup", ctx.String("address"))
+		logger.FatalIf(errInvalidArgument, "Invalid --address=\"%s\", port '0' is not allowed in a distributed erasure coded setup", ctxt.Addr)
 	}
 
 	globalLocalNodeName = GetLocalPeer(globalEndpoints, globalMinioHost, globalMinioPort)
@@ -270,7 +324,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	globalLocalNodeNameHex = hex.EncodeToString(nodeNameSum[:])
 
 	// Initialize, see which NIC the service is running on, and save it as global value
-	setGlobalInternodeInterface(ctx.String("interface"))
+	setGlobalInternodeInterface(ctxt.Interface)
 
 	// allow transport to be HTTP/1.1 for proxying.
 	globalProxyTransport = NewCustomHTTPProxyTransport()()
@@ -289,8 +343,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	})
 
 	globalTCPOptions = xhttp.TCPOptions{
-		UserTimeout: int(ctx.Duration("conn-user-timeout").Milliseconds()),
-		Interface:   ctx.String("interface"),
+		UserTimeout: int(ctxt.UserTimeout.Milliseconds()),
+		Interface:   ctxt.Interface,
 	}
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
@@ -299,13 +353,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(xhttp.CheckPortAvailability(globalMinioHost, globalMinioPort, globalTCPOptions), "Unable to start the server")
 
-	globalConnReadDeadline = ctx.Duration("conn-read-deadline")
-	globalConnWriteDeadline = ctx.Duration("conn-write-deadline")
-}
-
-func serverHandleEnvVars() {
-	// Handle common environment variables.
-	handleCommonEnvVars()
+	globalConnReadDeadline = ctxt.ConnReadDeadline
+	globalConnWriteDeadline = ctxt.ConnWriteDeadline
 }
 
 var globalHealStateLK sync.RWMutex
@@ -321,7 +370,7 @@ func initAllSubsystems(ctx context.Context) {
 	globalNotificationSys = NewNotificationSys(globalEndpoints)
 
 	// Create new notification system
-	globalEventNotifier = NewEventNotifier()
+	globalEventNotifier = NewEventNotifier(GlobalContext)
 
 	// Create new bucket metadata system.
 	if globalBucketMetadataSys == nil {
@@ -590,14 +639,24 @@ func serverMain(ctx *cli.Context) {
 	// Always load ENV variables from files first.
 	loadEnvVarsFromFiles()
 
-	// Handle all server command args.
+	// Handle all server command args and build the disks layout
 	bootstrapTrace("serverHandleCmdArgs", func() {
-		serverHandleCmdArgs(ctx)
+		err := buildServerCtxt(ctx, &globalServerCtxt)
+		logger.FatalIf(err, "Unable to prepare the list of endpoints")
+
+		serverHandleCmdArgs(globalServerCtxt)
 	})
 
+	// DNS cache subsystem to reduce outgoing DNS requests
+	runDNSCache(ctx)
+
 	// Handle all server environment vars.
 	serverHandleEnvVars()
 
+	// Load the root credentials from the shell environment or from
+	// the config file if not defined, set the default one.
+	loadRootCredentials()
+
 	// Initialize globalConsoleSys system
 	bootstrapTrace("newConsoleLogger", func() {
 		globalConsoleSys = NewConsoleLogger(GlobalContext)
@@ -637,7 +696,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Check for updates in non-blocking manner.
 	go func() {
-		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
+		if !globalServerCtxt.Quiet && !globalInplaceUpdateDisabled {
 			// Check for new updates from dl.min.io.
 			bootstrapTrace("checkUpdate", func() {
 				checkUpdate(getMinioMode())
@@ -683,9 +742,9 @@ func serverMain(ctx *cli.Context) {
 		httpServer := xhttp.NewServer(getServerListenAddrs()).
 			UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 			UseTLSConfig(newTLSConfig(getCert)).
-			UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
-			UseIdleTimeout(ctx.Duration("idle-timeout")).
-			UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
+			UseShutdownTimeout(globalServerCtxt.ShutdownTimeout).
+			UseIdleTimeout(globalServerCtxt.IdleTimeout).
+			UseReadHeaderTimeout(globalServerCtxt.ReadHeaderTimeout).
 			UseBaseContext(GlobalContext).
 			UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
 			UseTCPOptions(globalTCPOptions)
@@ -779,7 +838,7 @@ func serverMain(ctx *cli.Context) {
 			logger.LogIf(GlobalContext, err)
 		}
 
-		if !globalCLIContext.StrictS3Compat {
+		if !globalServerCtxt.StrictS3Compat {
 			logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
 		}
 	})
@@ -813,16 +872,16 @@ func serverMain(ctx *cli.Context) {
 		}
 
 		// if we see FTP args, start FTP if possible
-		if len(ctx.StringSlice("ftp")) > 0 {
+		if len(globalServerCtxt.FTP) > 0 {
 			bootstrapTrace("go startFTPServer", func() {
-				go startFTPServer(ctx)
+				go startFTPServer(globalServerCtxt.FTP)
 			})
 		}
 
 		// If we see SFTP args, start SFTP if possible
-		if len(ctx.StringSlice("sftp")) > 0 {
-			bootstrapTrace("go startFTPServer", func() {
-				go startSFTPServer(ctx)
+		if len(globalServerCtxt.SFTP) > 0 {
+			bootstrapTrace("go startSFTPServer", func() {
+				go startSFTPServer(globalServerCtxt.SFTP)
 			})
 		}
 	}()

cmd/server-main_test.go

@@ -23,6 +23,57 @@ import (
 	"testing"
 )
 
+func TestServerConfigFile(t *testing.T) {
+	for _, testcase := range []struct {
+		config      string
+		expectedErr bool
+		hash        string
+	}{
+		{
+			config:      "testdata/config/1.yaml",
+			expectedErr: false,
+			hash:        "hash:02bf70285dc71f76",
+		},
+		{
+			config:      "testdata/config/2.yaml",
+			expectedErr: false,
+			hash:        "hash:676d2da00f71f205",
+		},
+		{
+			config:      "testdata/config/invalid.yaml",
+			expectedErr: true,
+		},
+		{
+			config:      "testdata/config/invalid-types.yaml",
+			expectedErr: true,
+		},
+		{
+			config:      "testdata/config/invalid-disks.yaml",
+			expectedErr: true,
+		},
+	} {
+		testcase := testcase
+		t.Run(testcase.config, func(t *testing.T) {
+			sctx := &serverCtxt{}
+			err := mergeServerCtxtFromConfigFile(testcase.config, sctx)
+			if testcase.expectedErr && err == nil {
+				t.Error("expected failure, got success")
+			}
+			if !testcase.expectedErr && err != nil {
+				t.Error("expected success, got failure", err)
+			}
+			if err == nil {
+				if len(sctx.Layout.pools) != 2 {
+					t.Error("expected parsed pools to be 2, not", len(sctx.Layout.pools))
+				}
+				if sctx.Layout.pools[0].cmdline != testcase.hash {
+					t.Error("expected hash", testcase.hash, "got", sctx.Layout.pools[0].cmdline)
+				}
+			}
+		})
+	}
+}
+
 // Tests initializing new object layer.
 func TestNewObjectLayer(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())

cmd/server-startup-msg.go

@@ -121,7 +121,7 @@ func printServerCommonMsg(apiEndpoints []string) {
 
 	// Colorize the message and print.
 	logger.Info(color.Blue("S3-API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
-	if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+	if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON) {
 		logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
 		logger.Info(color.Blue("RootPass: ") + color.Bold("%s \n", cred.SecretKey))
 		if region != "" {
@@ -132,7 +132,7 @@ func printServerCommonMsg(apiEndpoints []string) {
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
 		logger.Info(color.Blue("Console: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
-		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+		if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON) {
 			logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
 			logger.Info(color.Blue("RootPass: ") + color.Bold("%s ", cred.SecretKey))
 		}
@@ -187,7 +187,7 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	const mcQuickStartGuide = "https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart"
 
 	// Configure 'mc', following block prints platform specific information for minio client.
-	if color.IsTerminal() && !globalCLIContext.Anonymous {
+	if color.IsTerminal() && !globalServerCtxt.Anonymous {
 		logger.Info(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
 		mcMessage := fmt.Sprintf("$ mc alias set '%s' '%s' '%s' '%s'", alias,
 			endPoint, cred.AccessKey, cred.SecretKey)

cmd/server_test.go

@@ -137,7 +137,7 @@ func TestServerSuite(t *testing.T) {
 		// Init and run test on ErasureSet backend.
 		{serverType: "ErasureSet", signer: signerV4},
 	}
-	globalCLIContext.StrictS3Compat = true
+	globalServerCtxt.StrictS3Compat = true
 	for i, testCase := range testCases {
 		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
 			runAllTests(testCase, &check{t, testCase.serverType})

cmd/sftp-server-driver.go

@@ -93,55 +93,66 @@ func (f *sftpDriver) getMinIOClient() (*minio.Client, error) {
 		return nil, errNoSuchUser
 	}
 	if !ok && globalIAMSys.LDAPConfig.Enabled() {
-		targetUser, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(f.AccessKey())
-		if err != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), f.AccessKey())
+		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
 			return nil, err
 		}
-		expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
-		if err != nil {
-			return nil, err
+		var mcreds *credentials.Credentials
+		if errors.Is(err, errNoSuchServiceAccount) {
+			targetUser, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(f.AccessKey())
+			if err != nil {
+				return nil, err
+			}
+			expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
+			if err != nil {
+				return nil, err
+			}
+			claims := make(map[string]interface{})
+			claims[expClaim] = UTCNow().Add(expiryDur).Unix()
+			for k, v := range f.permissions.CriticalOptions {
+				claims[k] = v
+			}
+
+			cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
+			if err != nil {
+				return nil, err
+			}
+
+			// Set the parent of the temporary access key, this is useful
+			// in obtaining service accounts by this cred.
+			cred.ParentUser = targetUser
+
+			// Set this value to LDAP groups, LDAP user can be part
+			// of large number of groups
+			cred.Groups = targetGroups
+
+			// Set the newly generated credentials, policyName is empty on purpose
+			// LDAP policies are applied automatically using their ldapUser, ldapGroups
+			// mapping.
+			updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
+			if err != nil {
+				return nil, err
+			}
+
+			// Call hook for site replication.
+			logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+				Type: madmin.SRIAMItemSTSAcc,
+				STSCredential: &madmin.SRSTSCredential{
+					AccessKey:    cred.AccessKey,
+					SecretKey:    cred.SecretKey,
+					SessionToken: cred.SessionToken,
+					ParentUser:   cred.ParentUser,
+				},
+				UpdatedAt: updatedAt,
+			}))
+
+			mcreds = credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken)
+		} else {
+			mcreds = credentials.NewStaticV4(sa.Credentials.AccessKey, sa.Credentials.SecretKey, "")
 		}
-		claims := make(map[string]interface{})
-		claims[expClaim] = UTCNow().Add(expiryDur).Unix()
-		for k, v := range f.permissions.CriticalOptions {
-			claims[k] = v
-		}
-
-		cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
-		if err != nil {
-			return nil, err
-		}
-
-		// Set the parent of the temporary access key, this is useful
-		// in obtaining service accounts by this cred.
-		cred.ParentUser = targetUser
-
-		// Set this value to LDAP groups, LDAP user can be part
-		// of large number of groups
-		cred.Groups = targetGroups
-
-		// Set the newly generated credentials, policyName is empty on purpose
-		// LDAP policies are applied automatically using their ldapUser, ldapGroups
-		// mapping.
-		updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
-		if err != nil {
-			return nil, err
-		}
-
-		// Call hook for site replication.
-		logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
-			Type: madmin.SRIAMItemSTSAcc,
-			STSCredential: &madmin.SRSTSCredential{
-				AccessKey:    cred.AccessKey,
-				SecretKey:    cred.SecretKey,
-				SessionToken: cred.SessionToken,
-				ParentUser:   cred.ParentUser,
-			},
-			UpdatedAt: updatedAt,
-		}))
 
 		return minio.New(f.endpoint, &minio.Options{
-			Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+			Creds:     mcreds,
 			Secure:    globalIsTLS,
 			Transport: globalRemoteFTPClientTransport,
 		})

cmd/sftp-server.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"crypto/subtle"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -27,7 +28,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/cli"
 	"github.com/minio/minio/internal/logger"
 	xsftp "github.com/minio/pkg/v2/sftp"
 	"github.com/pkg/sftp"
@@ -53,9 +53,7 @@ func (s *sftpLogger) Error(tag xsftp.LogType, err error) {
 	}
 }
 
-func startSFTPServer(c *cli.Context) {
-	args := c.StringSlice("sftp")
-
+func startSFTPServer(args []string) {
 	var (
 		port          int
 		publicIP      string
@@ -110,21 +108,36 @@ func startSFTPServer(c *cli.Context) {
 	sshConfig := &ssh.ServerConfig{
 		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
 			if globalIAMSys.LDAPConfig.Enabled() {
-				targetUser, targetGroups, err := globalIAMSys.LDAPConfig.Bind(c.User(), string(pass))
-				if err != nil {
+				sa, _, err := globalIAMSys.getServiceAccount(context.Background(), c.User())
+				if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
 					return nil, err
 				}
-				ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
-				if len(ldapPolicies) == 0 {
-					return nil, errAuthentication
+				if errors.Is(err, errNoSuchServiceAccount) {
+					targetUser, targetGroups, err := globalIAMSys.LDAPConfig.Bind(c.User(), string(pass))
+					if err != nil {
+						return nil, err
+					}
+					ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
+					if len(ldapPolicies) == 0 {
+						return nil, errAuthentication
+					}
+					return &ssh.Permissions{
+						CriticalOptions: map[string]string{
+							ldapUser:  targetUser,
+							ldapUserN: c.User(),
+						},
+						Extensions: make(map[string]string),
+					}, nil
 				}
-				return &ssh.Permissions{
-					CriticalOptions: map[string]string{
-						ldapUser:  targetUser,
-						ldapUserN: c.User(),
-					},
-					Extensions: make(map[string]string),
-				}, nil
+				if subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), pass) == 1 {
+					return &ssh.Permissions{
+						CriticalOptions: map[string]string{
+							"accessKey": c.User(),
+						},
+						Extensions: make(map[string]string),
+					}, nil
+				}
+				return nil, errAuthentication
 			}
 
 			ui, ok := globalIAMSys.GetUser(context.Background(), c.User())

cmd/signature-v4-utils.go

@@ -79,7 +79,7 @@ func skipContentSha256Cksum(r *http.Request) bool {
 		// We return true only in situations when
 		// deployment has asked MinIO to allow for
 		// such broken clients and content-length > 0.
-		return r.ContentLength > 0 && !globalCLIContext.StrictS3Compat
+		return r.ContentLength > 0 && !globalServerCtxt.StrictS3Compat
 	}
 	return false
 }

cmd/site-replication.go

@@ -3724,6 +3724,10 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			}
 
 			if meta.lifecycleConfig != nil && meta.lifecycleConfig.HasExpiry() {
+				var updatedAt time.Time
+				if meta.lifecycleConfig.ExpiryUpdatedAt != nil {
+					updatedAt = *meta.lifecycleConfig.ExpiryUpdatedAt
+				}
 				for _, rule := range meta.lifecycleConfig.Rules {
 					if !rule.Expiration.IsNull() || !rule.NoncurrentVersionExpiration.IsNull() {
 						// copy the non transition details of the rule
@@ -3731,7 +3735,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 						if err != nil {
 							return info, errSRBackendIssue(err)
 						}
-						allRules[rule.ID] = madmin.ILMExpiryRule{ILMRule: string(ruleData), Bucket: bucket, UpdatedAt: *(meta.lifecycleConfig.ExpiryUpdatedAt)}
+						allRules[rule.ID] = madmin.ILMExpiryRule{ILMRule: string(ruleData), Bucket: bucket, UpdatedAt: updatedAt}
 					}
 				}
 			}

cmd/storage-rest-server.go

@@ -1348,6 +1348,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			if !endpoint.IsLocal {
 				continue
 			}
+
 			driveHandlers[pool][set] = &storageRESTServer{}
 			server := driveHandlers[pool][set]
 
@@ -1392,15 +1393,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 				Handle:      server.WalkDirHandler,
 				OutCapacity: 1,
 			}), "unable to register handler")
-		}
-	}
 
-	for pool, serverPool := range endpointServerPools {
-		for set, endpoint := range serverPool.Endpoints {
-			if !endpoint.IsLocal {
-				continue
-			}
-			createStorage := func(pool, set int, endpoint Endpoint) bool {
+			createStorage := func(server *storageRESTServer) bool {
 				xl, err := newXLStorage(endpoint, false)
 				if err != nil {
 					// if supported errors don't fail, we proceed to
@@ -1410,21 +1404,22 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 				}
 				storage := newXLStorageDiskIDCheck(xl, true)
 				storage.SetDiskID(xl.diskID)
-				driveHandlers[pool][set].setStorage(storage)
+				server.setStorage(storage)
 				return true
 			}
-			if createStorage(pool, set, endpoint) {
+			if createStorage(server) {
 				continue
 			}
 			// Start async goroutine to create storage.
-			go func(pool, set int, endpoint Endpoint) {
+			go func(server *storageRESTServer) {
 				for {
-					time.Sleep(time.Minute)
-					if createStorage(pool, set, endpoint) {
+					time.Sleep(5 * time.Second)
+					if createStorage(server) {
 						return
 					}
 				}
-			}(pool, set, endpoint)
+			}(server)
+
 		}
 	}
 }

cmd/testdata/config/1.yaml

@@ -0,0 +1,23 @@
+version: v1
+address: ':9000'
+console-address: ':9001'
+certs-dir: '/home/user/.minio/certs/'
+pools: # Specify the nodes and drives with pools
+  -
+        - 'https://server-example-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server{1...2}-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool1:9000/mnt/disk{1...4}/'
+  -
+        - 'https://server-example-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server{1...2}-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool2:9000/mnt/disk{1...4}/'
+
+options:
+  ftp: # settings for MinIO to act as an ftp server
+        address: ':8021'
+        passive-port-range: '30000-40000'
+  sftp: # settings for MinIO to act as an sftp server
+        address: ':8022'
+        ssh-private-key: '/home/user/.ssh/id_rsa'

cmd/testdata/config/2.yaml

@@ -0,0 +1,23 @@
+version: v1
+address: ':9000'
+console-address: ':9001'
+certs-dir: '/home/user/.minio/certs/'
+pools: # Specify the nodes and drives with pools
+  -
+        - 'https://server-example-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server1-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool1:9000/mnt/disk{1...4}/'
+  -
+        - 'https://server-example-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server1-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool2:9000/mnt/disk{1...4}/'
+
+options:
+  ftp: # settings for MinIO to act as an ftp server
+        address: ':8021'
+        passive-port-range: '30000-40000'
+  sftp: # settings for MinIO to act as an sftp server
+        address: ':8022'
+        ssh-private-key: '/home/user/.ssh/id_rsa'

cmd/testdata/config/invalid-disks.yaml

@@ -0,0 +1,23 @@
+version: v1
+address: ':9000'
+console-address: ':9001'
+certs-dir: '/home/user/.minio/certs/'
+pools: # Specify the nodes and drives with pools
+  -
+        - 'https://server-example-pool1:9000/mnt/disk1/'
+        - 'https://server1-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool1:9000/mnt/disk{1...4}/'
+  -
+        - 'https://server-example-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server1-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool2:9000/mnt/disk{1...4}/'
+
+options:
+  ftp: # settings for MinIO to act as an ftp server
+        address: ':8021'
+        passive-port-range: '30000-40000'
+  sftp: # settings for MinIO to act as an sftp server
+        address: ':8022'
+        ssh-private-key: '/home/user/.ssh/id_rsa'

cmd/testdata/config/invalid-types.yaml

@@ -0,0 +1,23 @@
+version: v1
+address: ':9000'
+console-address: ':9001'
+certs-dir: '/home/user/.minio/certs/'
+pools: # Specify the nodes and drives with pools
+  -
+        - '/mnt/disk{1...4}/'
+        - 'https://server1-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool1:9000/mnt/disk{1...4}/'
+  -
+        - 'https://server-example-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server1-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool2:9000/mnt/disk{1...4}/'
+
+options:
+  ftp: # settings for MinIO to act as an ftp server
+        address: ':8021'
+        passive-port-range: '30000-40000'
+  sftp: # settings for MinIO to act as an sftp server
+        address: ':8022'
+        ssh-private-key: '/home/user/.ssh/id_rsa'

cmd/testdata/config/invalid.yaml

@@ -0,0 +1,23 @@
+version:
+address: ':9000'
+console-address: ':9001'
+certs-dir: '/home/user/.minio/certs/'
+pools: # Specify the nodes and drives with pools
+  -
+        - 'https://server-example-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server1-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool1:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool1:9000/mnt/disk{1...4}/'
+  -
+        - 'https://server-example-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server1-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server3-pool2:9000/mnt/disk{1...4}/'
+        - 'https://server4-pool2:9000/mnt/disk{1...4}/'
+
+options:
+  ftp: # settings for MinIO to act as an ftp server
+        address: ':8021'
+        passive-port-range: '30000-40000'
+  sftp: # settings for MinIO to act as an sftp server
+        address: ':8022'
+        ssh-private-key: '/home/user/.ssh/id_rsa'

cmd/tier.go

@@ -148,12 +148,11 @@ func (config *TierConfigMgr) Remove(ctx context.Context, tier string) error {
 		return err
 	} else if inuse {
 		return errTierBackendNotEmpty
-	} else {
-		config.Lock()
-		delete(config.Tiers, tier)
-		delete(config.drivercache, tier)
-		config.Unlock()
 	}
+	config.Lock()
+	delete(config.Tiers, tier)
+	delete(config.drivercache, tier)
+	config.Unlock()
 	return nil
 }
 

cmd/update-notifier.go

@@ -41,7 +41,7 @@ func prepareUpdateMessage(downloadURL string, older time.Duration) string {
 	t := time.Time{}
 	newerThan := humanize.RelTime(t, t.Add(older), "before the latest release", "")
 
-	if globalCLIContext.JSON {
+	if globalServerCtxt.JSON {
 		return fmt.Sprintf("You are running an older version of MinIO released %s, update: %s", newerThan, downloadURL)
 	}
 

cmd/warm-backend-azure.go

@@ -26,8 +26,11 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/go-autorest/autorest/azure"
 	"github.com/minio/madmin-go/v3"
 )
 
@@ -108,8 +111,53 @@ func (az *warmBackendAzure) InUse(ctx context.Context) (bool, error) {
 	return false, nil
 }
 
+func newCredentialFromSP(conf madmin.TierAzure) (azblob.Credential, error) {
+	oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, conf.SPAuth.TenantID)
+	if err != nil {
+		return nil, err
+	}
+	spt, err := adal.NewServicePrincipalToken(*oauthConfig, conf.SPAuth.ClientID, conf.SPAuth.ClientSecret, azure.PublicCloud.ResourceIdentifiers.Storage)
+	if err != nil {
+		return nil, err
+	}
+
+	// Refresh obtains a fresh token
+	err = spt.Refresh()
+	if err != nil {
+		return nil, err
+	}
+
+	tc := azblob.NewTokenCredential(spt.Token().AccessToken, func(tc azblob.TokenCredential) time.Duration {
+		err := spt.Refresh()
+		if err != nil {
+			return 0
+		}
+		// set the new token value
+		tc.SetToken(spt.Token().AccessToken)
+
+		// get the next token before the current one expires
+		nextRenewal := float64(time.Until(spt.Token().Expires())) * 0.8
+		if nextRenewal <= 0 {
+			nextRenewal = float64(time.Second)
+		}
+
+		return time.Duration(nextRenewal)
+	})
+
+	return tc, nil
+}
+
 func newWarmBackendAzure(conf madmin.TierAzure, _ string) (*warmBackendAzure, error) {
-	credential, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
+	var (
+		credential azblob.Credential
+		err        error
+	)
+
+	if conf.IsSPEnabled() {
+		credential, err = newCredentialFromSP(conf)
+	} else {
+		credential, err = azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey)
+	}
 	if err != nil {
 		if _, ok := err.(base64.CorruptInputError); ok {
 			return nil, errors.New("invalid Azure credentials")

cmd/xl-storage-disk-id-check.go

@@ -78,8 +78,9 @@ const (
 
 // Detects change in underlying disk.
 type xlStorageDiskIDCheck struct {
-	totalErrsAvailability uint64 // Captures all data availability errors such as permission denied, faulty disk and timeout errors.
-	totalErrsTimeout      uint64 // Captures all timeout only errors
+	totalErrsTimeout      atomic.Uint64 // Captures all timeout only errors
+	totalErrsAvailability atomic.Uint64 // Captures all data availability errors such as permission denied, faulty disk and timeout errors.
+
 	// apiCalls should be placed first so alignment is guaranteed for atomic operations.
 	apiCalls     [storageMetricLast]uint64
 	apiLatencies [storageMetricLast]*lockedLastMinuteLatency
@@ -102,7 +103,7 @@ type xlStorageDiskIDCheck struct {
 
 func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
 	p.metricsCache.Once.Do(func() {
-		p.metricsCache.TTL = 1 * time.Second
+		p.metricsCache.TTL = 5 * time.Second
 		p.metricsCache.Update = func() (interface{}, error) {
 			diskMetric := DiskMetrics{
 				LastMinute: make(map[string]AccElem, len(p.apiLatencies)),
@@ -114,13 +115,19 @@ func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
 			for i := range p.apiCalls {
 				diskMetric.APICalls[storageMetric(i).String()] = atomic.LoadUint64(&p.apiCalls[i])
 			}
-			diskMetric.TotalErrorsAvailability = atomic.LoadUint64(&p.totalErrsAvailability)
-			diskMetric.TotalErrorsTimeout = atomic.LoadUint64(&p.totalErrsTimeout)
 			return diskMetric, nil
 		}
 	})
 	m, _ := p.metricsCache.Get()
-	return m.(DiskMetrics)
+	diskMetric := DiskMetrics{}
+	if m != nil {
+		diskMetric = m.(DiskMetrics)
+	}
+
+	// Do not need this value to be cached.
+	diskMetric.TotalErrorsTimeout = p.totalErrsTimeout.Load()
+	diskMetric.TotalErrorsAvailability = p.totalErrsAvailability.Load()
+	return diskMetric
 }
 
 // lockedLastMinuteLatency accumulates totals lockless for each second.
@@ -746,9 +753,9 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 			context.DeadlineExceeded,
 			context.Canceled,
 		}...) {
-			atomic.AddUint64(&p.totalErrsAvailability, 1)
+			p.totalErrsAvailability.Add(1)
 			if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
-				atomic.AddUint64(&p.totalErrsTimeout, 1)
+				p.totalErrsTimeout.Add(1)
 			}
 		}
 		p.apiLatencies[s].add(duration)

cmd/xl-storage.go

@@ -1005,8 +1005,10 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		return err
 	}
 
+	discard := true
+
 	var legacyJSON bool
-	buf, _, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), false)
+	buf, _, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), discard)
 	if err != nil {
 		if !errors.Is(err, errFileNotFound) {
 			return err
@@ -1016,7 +1018,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		legacy := s.formatLegacy
 		s.RUnlock()
 		if legacy {
-			buf, _, err = s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFileV1), false)
+			buf, _, err = s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFileV1), discard)
 			if err != nil {
 				return err
 			}
@@ -1163,7 +1165,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 	}
 
 	var legacyJSON bool
-	buf, _, err := s.readAllData(ctx, volume, volumeDir, pathJoin(filePath, xlStorageFormatFile), false)
+	buf, _, err := s.readAllData(ctx, volume, volumeDir, pathJoin(filePath, xlStorageFormatFile), true)
 	if err != nil {
 		if !errors.Is(err, errFileNotFound) {
 			return err
@@ -1401,7 +1403,7 @@ func (s *xlStorage) readRaw(ctx context.Context, volume, volumeDir, filePath str
 
 	xlPath := pathJoin(filePath, xlStorageFormatFile)
 	if readData {
-		buf, dmTime, err = s.readAllData(ctx, volume, volumeDir, xlPath, true)
+		buf, dmTime, err = s.readAllData(ctx, volume, volumeDir, xlPath, false)
 	} else {
 		buf, dmTime, err = s.readMetadataWithDMTime(ctx, xlPath)
 		if err != nil {
@@ -1421,7 +1423,7 @@ func (s *xlStorage) readRaw(ctx context.Context, volume, volumeDir, filePath str
 	s.RUnlock()
 
 	if err != nil && errors.Is(err, errFileNotFound) && legacy {
-		buf, dmTime, err = s.readAllData(ctx, volume, volumeDir, pathJoin(filePath, xlStorageFormatFileV1), true)
+		buf, dmTime, err = s.readAllData(ctx, volume, volumeDir, pathJoin(filePath, xlStorageFormatFileV1), false)
 		if err != nil {
 			return nil, time.Time{}, err
 		}
@@ -1538,14 +1540,14 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 			len(fi.Parts) == 1 {
 			partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
 			dataPath := pathJoin(volumeDir, path, fi.DataDir, partPath)
-			fi.Data, _, err = s.readAllData(ctx, volume, volumeDir, dataPath, true)
+			fi.Data, _, err = s.readAllData(ctx, volume, volumeDir, dataPath, false)
 			if err != nil {
 				return FileInfo{}, err
 			}
 		}
 	}
 
-	if !skipAccessChecks(volume) && !opts.Healing && fi.TransitionStatus == "" && !fi.InlineData() && len(fi.Data) == 0 && fi.DataDir != "" && fi.DataDir != emptyUUID {
+	if !skipAccessChecks(volume) && !opts.Healing && fi.TransitionStatus == "" && !fi.InlineData() && len(fi.Data) == 0 && fi.DataDir != "" && fi.DataDir != emptyUUID && fi.VersionPurgeStatus().Empty() {
 		// Verify if the dataDir is present or not when the data
 		// is not inlined to make sure we return correct errors
 		// during HeadObject().
@@ -1564,7 +1566,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 	return fi, nil
 }
 
-func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, filePath string, sync bool) (buf []byte, dmTime time.Time, err error) {
+func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, filePath string, discard bool) (buf []byte, dmTime time.Time, err error) {
 	if filePath == "" {
 		return nil, dmTime, errFileNotFound
 	}
@@ -1573,13 +1575,7 @@ func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, f
 		return nil, time.Time{}, ctx.Err()
 	}
 
-	odirectEnabled := globalAPIConfig.odirectEnabled() && s.oDirect
-	var f *os.File
-	if odirectEnabled && sync {
-		f, err = OpenFileDirectIO(filePath, readMode, 0o666)
-	} else {
-		f, err = OpenFile(filePath, readMode, 0o666)
-	}
+	f, err := OpenFile(filePath, readMode, 0o666)
 	if err != nil {
 		switch {
 		case osIsNotExist(err):
@@ -1614,23 +1610,20 @@ func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, f
 		}
 		return nil, dmTime, err
 	}
-	r := io.Reader(f)
-	var dr *xioutil.ODirectReader
-	if odirectEnabled {
-		dr = &xioutil.ODirectReader{
-			File:      f,
-			SmallFile: true,
-		}
-		defer dr.Close()
-		r = dr
-	} else {
-		defer f.Close()
+
+	if discard {
+		// This discard is mostly true for DELETEs
+		// so we need to make sure we do not keep
+		// page-cache references after.
+		defer disk.Fdatasync(f)
 	}
 
+	defer f.Close()
+
 	// Get size for precise allocation.
 	stat, err := f.Stat()
 	if err != nil {
-		buf, err = io.ReadAll(r)
+		buf, err = io.ReadAll(f)
 		return buf, dmTime, osErrToFileErr(err)
 	}
 	if stat.IsDir() {
@@ -1645,11 +1638,9 @@ func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, f
 	} else {
 		buf = make([]byte, sz)
 	}
-	if dr != nil {
-		dr.SmallFile = sz <= xioutil.BlockSizeSmall*2
-	}
+
 	// Read file...
-	_, err = io.ReadFull(r, buf)
+	_, err = io.ReadFull(f, buf)
 
 	return buf, stat.ModTime().UTC(), osErrToFileErr(err)
 }
@@ -2284,6 +2275,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			errDiskNotFound,
 			errUnformattedDisk,
 			errMaxVersionsExceeded,
+			errFileAccessDenied,
 		}
 		if err != nil && !IsErr(err, ignoredErrs...) && !contextCanceled(ctx) {
 			// Only log these errors if context is not yet canceled.
@@ -2291,7 +2283,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 				s.drivePath,
 				srcVolume, srcPath,
 				dstVolume, dstPath,
-				err), "xl-storage-rename-data-"+srcVolume+"-"+dstVolume)
+				err), "xl-storage-rename-data-"+dstVolume)
 		}
 		if err == nil && s.globalSync {
 			globalSync()
@@ -2780,7 +2772,7 @@ func (s *xlStorage) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp
 			if req.MetadataOnly {
 				data, mt, err = s.readMetadataWithDMTime(ctx, fullPath)
 			} else {
-				data, mt, err = s.readAllData(ctx, req.Bucket, volumeDir, fullPath, false)
+				data, mt, err = s.readAllData(ctx, req.Bucket, volumeDir, fullPath, true)
 			}
 			return err
 		}); err != nil {
@@ -2875,7 +2867,7 @@ func (s *xlStorage) CleanAbandonedData(ctx context.Context, volume string, path
 	}
 	baseDir := pathJoin(volumeDir, path+slashSeparator)
 	metaPath := pathutil.Join(baseDir, xlStorageFormatFile)
-	buf, _, err := s.readAllData(ctx, volume, volumeDir, metaPath, false)
+	buf, _, err := s.readAllData(ctx, volume, volumeDir, metaPath, true)
 	if err != nil {
 		return err
 	}

docs/distributed/CONFIG.md

@@ -0,0 +1,67 @@
+## MinIO configuration YAML
+
+MinIO now supports starting the server arguments and configuration via a YAML configuration file. This YAML configuration describes everything that can be configured in a MinIO setup, such as '--address', '--console-address' and command line arguments for the MinIO server.
+
+Historically everything to MinIO was provided via command arguments for the hostnames and the drives via an ellipses syntax such as `minio server http://host{1...4}/disk{1...4}` this requirement added an additional burden to have sequential hostnames for us to make sure that we can provide horizontal distribution, however we have come across situations where sometimes this is not feasible and there are no easier alternatives without modifying /etc/hosts on the host system as root user.  Many times in airgapped deployments this is not allowed or requires audits and approvals.
+
+MinIO server configuration file allows users to provide topology that allows for heterogenous hostnames, allowing MinIO to deployed in pre-existing environments without any further OS level configurations.
+
+### Usage
+
+```
+minio server --config config.yaml
+```
+
+Lets you start MinIO server with all inputs to start MinIO server provided via this configuration file, once the configuration file is provided all other pre-existing values on disk for configuration are overriden by the new values set in this configuration file.
+
+Following is an example YAML configuration structure.
+```
+version: v1
+address: ':9000'
+rootUser: 'minioadmin'
+rootPassword: 'pBU94AGAY85e'
+console-address: ':9001'
+certs-dir: '/home/user/.minio/certs/'
+pools: # Specify the nodes and drives with pools
+  -
+	- 'https://server-example-pool1:9000/mnt/disk{1...4}/'
+	- 'https://server{1...2}-pool1:9000/mnt/disk{1...4}/'
+	- 'https://server3-pool1:9000/mnt/disk{1...4}/'
+	- 'https://server4-pool1:9000/mnt/disk{1...4}/'
+  -
+	- 'https://server-example-pool2:9000/mnt/disk{1...4}/'
+	- 'https://server{1...2}-pool2:9000/mnt/disk{1...4}/'
+	- 'https://server3-pool2:9000/mnt/disk{1...4}/'
+	- 'https://server4-pool2:9000/mnt/disk{1...4}/'
+
+...
+
+options:
+  ftp: # settings for MinIO to act as an ftp server
+	address: ':8021'
+	passive-port-range: '30000-40000'
+  sftp: # settings for MinIO to act as an sftp server
+	address: ':8022'
+	ssh-private-key: '/home/user/.ssh/id_rsa'
+```
+
+### Things to know
+
+- Fields such as `version` and `pools` are mandatory, however all other fields are optional.
+- Each pool expects a minimum of 2 nodes per pool, and unique non-repeating hosts for each argument.
+- Each pool expects each host in this pool has the same number of drives specified as any other host.
+- Mixing `local-path` and `distributed-path` is not allowed, doing so would cause MinIO to refuse starting the server.
+- Ellipses notation (e.g. `{1...10}`) or bracket notations are fully allowed (e.g. `{a,c,f}`) to have multiple entries in one line.
+
+> NOTE: MinIO environmental variables still take precedence over the `config.yaml` file, however `config.yaml` is preffered over MinIO internal config KV settings via `mc admin config set alias/ <sub-system>`.
+
+### TODO
+
+In subsequent releases we are planning to extend this to provide things like
+
+- Reload() of MinIO server arguments without fully restarting the process.
+
+- Expanding 1 node at a time by automating the process of creating a new pool
+  and decommissioning to provide a functionality that smaller deployments
+  care about.
+

docs/distributed/decom.sh

@@ -102,8 +102,13 @@ fi
 
 ./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
 
+count=0
 until $(./mc admin decom status myminio/ | grep -q Complete); do
 	echo "waiting for decom to finish..."
+	count=$((count + 1))
+	if [ ${count} -eq 120 ]; then
+		./mc cat /tmp/expanded.log
+	fi
 	sleep 1
 done
 

docs/distributed/distributed-from-config-file.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+set -e
+
+cleanup() {
+	echo "Cleaning up instances of MinIO"
+	pkill minio || true
+	pkill -9 minio || true
+	rm -rf /tmp/xl/ || true
+	rm -rf /tmp/minio.configfile.{1,2,3,4} || true
+}
+
+cleanup
+
+unset MINIO_KMS_KES_CERT_FILE
+unset MINIO_KMS_KES_KEY_FILE
+unset MINIO_KMS_KES_ENDPOINT
+unset MINIO_KMS_KES_KEY_NAME
+
+export MINIO_CI_CD=1
+
+if [ ! -f ./mc ]; then
+	os="$(uname -s)"
+	arch="$(uname -m)"
+	wget -O mc https://dl.minio.io/client/mc/release/${os,,}-${arch,,}/mc &&
+		chmod +x mc
+fi
+
+for i in $(seq 1 4); do
+	s3Port="$((9000 + i))"
+	consolePort="$((s3Port + 1000))"
+
+	cat <<EOF >/tmp/minio.configfile.$i
+version: v1
+address: ':${s3Port}'
+console-address: ':${consolePort}'
+rootUser: 'minr0otUS2r'
+rootPassword: 'pBU94AGAY85e'
+pools: # Specify the nodes and drives with pools
+  -
+     - 'http://localhost:9001/tmp/xl/node9001/mnt/disk{1...4}/'
+     - 'http://localhost:9002/tmp/xl/node9002/mnt/disk{1,2,3,4}/'
+  -
+     - 'http://localhost:9003/tmp/xl/node9003/mnt/disk{1...4}/'
+     - 'http://localhost:9004/tmp/xl/node9004/mnt/disk1/'
+     - 'http://localhost:9004/tmp/xl/node9004/mnt/disk2/'
+     - 'http://localhost:9004/tmp/xl/node9004/mnt/disk3/'
+     - 'http://localhost:9004/tmp/xl/node9004/mnt/disk4/'
+EOF
+done
+
+minio server --config /tmp/minio.configfile.1 >/tmp/minio1_1.log 2>&1 &
+site1_pid=$!
+minio server --config /tmp/minio.configfile.2 >/tmp/minio2_1.log 2>&1 &
+site2_pid=$!
+minio server --config /tmp/minio.configfile.3 >/tmp/minio3_1.log 2>&1 &
+site3_pid=$!
+minio server --config /tmp/minio.configfile.4 >/tmp/minio4_1.log 2>&1 &
+site4_pid=$!
+
+sleep 5
+
+export MC_HOST_minio1=http://minr0otUS2r:pBU94AGAY85e@localhost:9001
+export MC_HOST_minio3=http://minr0otUS2r:pBU94AGAY85e@localhost:9003
+
+./mc ready minio1
+./mc ready minio3
+
+./mc mb minio1/testbucket
+# copy large upload to newbucket on minio1
+truncate -s 17M lrgfile
+expected_checksum=$(cat ./lrgfile | md5sum)
+
+./mc cp ./lrgfile minio1/testbucket
+
+actual_checksum=$(./mc cat minio3/testbucket/lrgfile | md5sum)
+
+if [ "${expected_checksum}" != "${actual_checksum}" ]; then
+	echo "unexpected object checksum, expected: ${expected_checksum} got: ${actual_checksum}"
+	exit
+fi
+
+# Compare the difference of the list of disks and their location, with the below exected output
+diff <(./mc admin info minio1 --json | jq -r '.info.servers[].drives[] | "\(.pool_index),\(.set_index),\(.disk_index) \(.endpoint)"' | sort) <(
+	cat <<EOF
+0,0,0 http://localhost:9001/tmp/xl/node9001/mnt/disk1
+0,0,1 http://localhost:9002/tmp/xl/node9002/mnt/disk1
+0,0,2 http://localhost:9001/tmp/xl/node9001/mnt/disk2
+0,0,3 http://localhost:9002/tmp/xl/node9002/mnt/disk2
+0,0,4 http://localhost:9001/tmp/xl/node9001/mnt/disk3
+0,0,5 http://localhost:9002/tmp/xl/node9002/mnt/disk3
+0,0,6 http://localhost:9001/tmp/xl/node9001/mnt/disk4
+0,0,7 http://localhost:9002/tmp/xl/node9002/mnt/disk4
+1,0,0 http://localhost:9003/tmp/xl/node9003/mnt/disk1
+1,0,1 http://localhost:9004/tmp/xl/node9004/mnt/disk1
+1,0,2 http://localhost:9003/tmp/xl/node9003/mnt/disk2
+1,0,3 http://localhost:9004/tmp/xl/node9004/mnt/disk2
+1,0,4 http://localhost:9003/tmp/xl/node9003/mnt/disk3
+1,0,5 http://localhost:9004/tmp/xl/node9004/mnt/disk3
+1,0,6 http://localhost:9003/tmp/xl/node9003/mnt/disk4
+1,0,7 http://localhost:9004/tmp/xl/node9004/mnt/disk4
+EOF
+)
+
+cleanup

docs/metrics/prometheus/grafana/minio-bucket.json

@@ -364,7 +364,7 @@
       "title": "S3 API Request 4xx Error Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -456,7 +456,7 @@
       "title": "Inflight Requests Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -548,7 +548,7 @@
       "title": "Total Requests Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -640,7 +640,7 @@
       "title": "Total Data Sent Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -732,7 +732,7 @@
       "title": "Usage Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -824,7 +824,7 @@
       "title": "Objects Increase Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -916,7 +916,7 @@
       "title": "Versions Increase Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1008,7 +1008,7 @@
       "title": "Delete Markers Increase Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1189,7 +1189,7 @@
       "title": "Total Data Received Rate",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1269,7 +1269,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (bucket) (rate(minio_bucket_replication_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))",
+          "expr": "sum by (bucket) (minio_bucket_replication_received_bytes{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "Data Received [{{bucket}}]",
@@ -1278,10 +1278,10 @@
       ],
       "thresholds": [],
       "timeRegions": [],
-      "title": "Replication Data Received Rate ",
+      "title": "Replication Data Received",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1361,7 +1361,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": true,
-          "expr": "sum by (bucket) (rate(minio_bucket_replication_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))",
+          "expr": "sum by (bucket) (minio_bucket_replication_sent_bytes{job=\"$scrape_jobs\"})",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "Replication Data Sent [{{bucket}}]",
@@ -1370,10 +1370,10 @@
       ],
       "thresholds": [],
       "timeRegions": [],
-      "title": "Replication Data Sent Rate ",
+      "title": "Replication Data Sent",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1557,7 +1557,7 @@
       "title": "Replication Failed Objects",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1649,7 +1649,7 @@
       "title": "Replicated In Objects",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1741,7 +1741,7 @@
       "title": "Replicated Out Objects",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1833,7 +1833,7 @@
       "title": "Last Hour Failed Size",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -1925,7 +1925,7 @@
       "title": "Last Hour Failed Objects",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -2017,7 +2017,7 @@
       "title": "Last Minute Failed Size",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -2109,7 +2109,7 @@
       "title": "Last Minute Failed Objects",
       "tooltip": {
         "shared": true,
-        "sort": 0,
+        "sort": 2,
         "value_type": "individual"
       },
       "type": "graph",
@@ -2272,4 +2272,4 @@
   "uid": "TgmJnqnnk2",
   "version": 3,
   "weekStart": ""
-}
+}

docs/metrics/prometheus/list.md

@@ -14,225 +14,216 @@ For deployments behind a load balancer, use the load balancer hostname instead o
 
 ## Audit Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_audit_failed_messages`                 | Total number of messages that failed to send since start.                                                       |
-| `minio_audit_target_queue_length`             | Number of unsent messages in queue for target.                                                                  |
-| `minio_audit_total_messages`                  | Total number of messages sent since start.                                                                      |
-
-## Cache Metrics
-
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cache_hits_total`                      | Total number of drive cache hits.                                                                               |
-| `minio_cache_missed_total`                    | Total number of drive cache misses.                                                                             |
-| `minio_cache_sent_bytes`                      | Total number of bytes served from cache.                                                                        |
-| `minio_cache_total_bytes`                     | Total size of cache drive in bytes.                                                                             |
-| `minio_cache_usage_info`                      | Total percentage cache usage, value of 1 indicates high and 0 low, label level is set as well.                  |
-| `minio_cache_used_bytes`                      | Current cache usage in bytes.                                                                                   |
+| Name                              | Description                                               |
+|:----------------------------------|:----------------------------------------------------------|
+| `minio_audit_failed_messages`     | Total number of messages that failed to send since start. |
+| `minio_audit_target_queue_length` | Number of unsent messages in queue for target.            |
+| `minio_audit_total_messages`      | Total number of messages sent since start.                |
 
 ## Cluster Capacity Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cluster_capacity_raw_free_bytes`       | Total free capacity online in the cluster.                                                                      |
-| `minio_cluster_capacity_raw_total_bytes`      | Total capacity online in the cluster.                                                                           |
-| `minio_cluster_capacity_usable_free_bytes`    | Total free usable capacity online in the cluster.                                                               |
-| `minio_cluster_capacity_usable_total_bytes`   | Total usable capacity online in the cluster.                                                                    |
-| `minio_cluster_objects_size_distribution`     | Distribution of object sizes across a cluster                                                                   |
-| `minio_cluster_objects_version_distribution`  | Distribution of object versions across a cluster                                                                |
-| `minio_cluster_usage_object_total`            | Total number of objects in a cluster                                                                            |
-| `minio_cluster_usage_total_bytes`             | Total cluster usage in bytes                                                                                    |
-| `minio_cluster_usage_version_total`           | Total number of versions (includes delete marker) in a cluster                                                  |
-| `minio_cluster_usage_deletemarker_total`      | Total number of delete markers in a cluster                                                                     |
-| `minio_cluster_usage_total_bytes`             | Total cluster usage in bytes                                                                                    |
-| `minio_cluster_buckets_total`                 | Total number of buckets in the cluster                                                                          |
+| Name                                         | Description                                                    |
+|:---------------------------------------------|:---------------------------------------------------------------|
+| `minio_cluster_capacity_raw_free_bytes`      | Total free capacity online in the cluster.                     |
+| `minio_cluster_capacity_raw_total_bytes`     | Total capacity online in the cluster.                          |
+| `minio_cluster_capacity_usable_free_bytes`   | Total free usable capacity online in the cluster.              |
+| `minio_cluster_capacity_usable_total_bytes`  | Total usable capacity online in the cluster.                   |
+| `minio_cluster_objects_size_distribution`    | Distribution of object sizes across a cluster                  |
+| `minio_cluster_objects_version_distribution` | Distribution of object versions across a cluster               |
+| `minio_cluster_usage_object_total`           | Total number of objects in a cluster                           |
+| `minio_cluster_usage_total_bytes`            | Total cluster usage in bytes                                   |
+| `minio_cluster_usage_version_total`          | Total number of versions (includes delete marker) in a cluster |
+| `minio_cluster_usage_deletemarker_total`     | Total number of delete markers in a cluster                    |
+| `minio_cluster_usage_total_bytes`            | Total cluster usage in bytes                                   |
+| `minio_cluster_buckets_total`                | Total number of buckets in the cluster                         |
 
 ## Cluster Drive Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cluster_drive_offline_total`           | Total drives offline in this cluster.                                                                           |
-| `minio_cluster_drive_online_total`            | Total drives online in this cluster.                                                                            |
-| `minio_cluster_drive_total`                   | Total drives in this cluster.                                                                                   |
+| Name                                | Description                           |
+|:------------------------------------|:--------------------------------------|
+| `minio_cluster_drive_offline_total` | Total drives offline in this cluster. |
+| `minio_cluster_drive_online_total`  | Total drives online in this cluster.  |
+| `minio_cluster_drive_total`         | Total drives in this cluster.         |
 
 ## Cluster ILM Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cluster_ilm_transitioned_bytes`        | Total bytes transitioned to a tier.                                                                             |
-| `minio_cluster_ilm_transitioned_objects`      | Total number of objects transitioned to a tier.                                                                 |
-| `minio_cluster_ilm_transitioned_versions`     | Total number of versions transitioned to a tier.                                                                |
+| Name                                      | Description                                      |
+|:------------------------------------------|:-------------------------------------------------|
+| `minio_cluster_ilm_transitioned_bytes`    | Total bytes transitioned to a tier.              |
+| `minio_cluster_ilm_transitioned_objects`  | Total number of objects transitioned to a tier.  |
+| `minio_cluster_ilm_transitioned_versions` | Total number of versions transitioned to a tier. |
 
 ## Cluster KMS Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cluster_kms_online`                    | Reports whether the KMS is online (1) or offline (0).                                                           |
-| `minio_cluster_kms_request_error`             | Number of KMS requests that failed due to some error. (HTTP 4xx status code).                                   |
-| `minio_cluster_kms_request_failure`           | Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code).                        |
-| `minio_cluster_kms_request_success`           | Number of KMS requests that succeeded.                                                                          |
-| `minio_cluster_kms_uptime`                    | The time the KMS has been up and running in seconds.                                                            |
+| Name                                | Description                                                                              |
+|:------------------------------------|:-----------------------------------------------------------------------------------------|
+| `minio_cluster_kms_online`          | Reports whether the KMS is online (1) or offline (0).                                    |
+| `minio_cluster_kms_request_error`   | Number of KMS requests that failed due to some error. (HTTP 4xx status code).            |
+| `minio_cluster_kms_request_failure` | Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code). |
+| `minio_cluster_kms_request_success` | Number of KMS requests that succeeded.                                                   |
+| `minio_cluster_kms_uptime`          | The time the KMS has been up and running in seconds.                                     |
 
 ## Cluster Health Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cluster_nodes_offline_total`           | Total number of MinIO nodes offline.                                                                            |
-| `minio_cluster_nodes_online_total`            | Total number of MinIO nodes online.                                                                             |
-| `minio_cluster_write_quorum`                  | Maximum write quorum across all pools and sets                                                                  |
-| `minio_cluster_health_status`                 | Get current cluster health status                                                                               |
+| Name                                | Description                                    |
+|:------------------------------------|:-----------------------------------------------|
+| `minio_cluster_nodes_offline_total` | Total number of MinIO nodes offline.           |
+| `minio_cluster_nodes_online_total`  | Total number of MinIO nodes online.            |
+| `minio_cluster_write_quorum`        | Maximum write quorum across all pools and sets |
+| `minio_cluster_health_status`       | Get current cluster health status              |
 
 ## Cluster Replication Metrics
 
 Metrics marked as ``Site Replication Only`` only populate on deployments with [Site Replication](https://min.io/docs/minio/linux/operations/install-deploy-manage/multi-site-replication.html) configurations.
 For deployments with [bucket](https://min.io/docs/minio/linux/administration/bucket-replication.html) or [batch](https://min.io/docs/minio/linux/administration/batch-framework.html#replicate) configurations, these metrics populate instead under the [Bucket Metrics](#bucket-metrics) endpoint.
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_cluster_replication_current_active_workers`                 | Total number of active replication workers                                                 |
-| `minio_cluster_replication_average_active_workers`                 | Average number of active replication workers                                               |
-| `minio_cluster_replication_max_active_workers`                 | Maximum number of active replication workers seen since server start                           |
-| `minio_cluster_replication_link_online`       | Reports whether the replication link is online (1) or offline (0).                                              |
-| `minio_cluster_replication_link_offline_duration_seconds` | Total duration of replication link being offline in seconds since last offline event|
-| `minio_cluster_replication_link_downtime_duration_seconds` | Total downtime of replication link in seconds since server start|
-| `minio_cluster_replication_average_link_latency_ms` | Average replication link latency in milliseconds                                                          |
-| `minio_cluster_replication_max_link_latency_ms`     | Maximum replication link latency in milliseconds seen since server start                                  |
-| `minio_cluster_replication_current_link_latency_ms` | Current replication link latency in milliseconds                                                          |
-| `minio_cluster_replication_current_transfer_rate` | Current replication transfer rate in bytes/sec                                                              |
-| `minio_cluster_replication_average_transfer_rate` | Average replication transfer rate in bytes/sec                                                              |
-| `minio_cluster_replication_max_transfer_rate`     | Maximum replication transfer rate in bytes/sec seen since server start                                      |
-| `minio_cluster_replication_last_minute_queued_count` | Total number of objects queued for replication in the last full minute                                   |
-| `minio_cluster_replication_last_minute_queued_bytes` | Total number of bytes queued for replication in the last full minute                                     |
-| `minio_cluster_replication_average_queued_count` | Average number of objects queued for replication since server start                                          |
-| `minio_cluster_replication_average_queued_bytes` | Average number of bytes queued for replication since server start                                            |
-| `minio_cluster_replication_max_queued_bytes`  | Maximum number of bytes queued for replication seen since server start                                          |
-| `minio_cluster_replication_max_queued_count`  | Maximum number of objects queued for replication seen since server start                                        |
-| `minio_cluster_replication_recent_backlog_count` | Total number of objects seen in replication backlog in the last 5 minutes                                    |
-| `minio_cluster_replication_last_minute_failed_bytes`           | Total number of bytes failed at least once to replicate in the last full minute.                        |
-| `minio_cluster_replication_last_minute_failed_count`           | Total number of objects which failed replication in the last full minute.                               |
-| `minio_cluster_replication_last_hour_failed_bytes` | (_Site Replication Only_) Total number of bytes failed at least once to replicate in the last full hour.                          |
-| `minio_cluster_replication_last_hour_failed_count` | (_Site Replication Only_) Total number of objects which failed replication in the last full hour.                                 |
-| `minio_cluster_replication_total_failed_bytes`     | (_Site Replication Only_) Total number of bytes failed at least once to replicate since server start.                            |
-| `minio_cluster_replication_total_failed_count`     | (_Site Replication Only_) Total number of objects which failed replication since server start.                                   |
-| `minio_cluster_replication_received_bytes`         | (_Site Replication Only_) Total number of bytes replicated to this cluster from another source cluster.     |
-| `minio_cluster_replication_received_count`         | (_Site Replication Only_) Total number of objects received by this cluster from another source cluster.     |
-| `minio_cluster_replication_sent_bytes`             | (_Site Replication Only_) Total number of bytes replicated to the target cluster. |                         |
-| `minio_cluster_replication_sent_count`             | (_Site Replication Only_) Total number of objects replicated to the target cluster.  |                        |
-| `minio_cluster_replication_credential_errors`      | (_Site Replication Only_) Total number of replication credential errors since server start                |
+| Name                                                       | Description                                                                                             
+|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------|
+| `minio_cluster_replication_current_active_workers`         | Total number of active replication workers                                                               |
+| `minio_cluster_replication_average_active_workers`         | Average number of active replication workers                                                             |
+| `minio_cluster_replication_max_active_workers`             | Maximum number of active replication workers seen since server start                                     |
+| `minio_cluster_replication_link_online`                    | Reports whether the replication link is online (1) or offline (0).                                       |
+| `minio_cluster_replication_link_offline_duration_seconds`  | Total duration of replication link being offline in seconds since last offline event                     |
+| `minio_cluster_replication_link_downtime_duration_seconds` | Total downtime of replication link in seconds since server start                                         |
+| `minio_cluster_replication_average_link_latency_ms`        | Average replication link latency in milliseconds                                                         |
+| `minio_cluster_replication_max_link_latency_ms`            | Maximum replication link latency in milliseconds seen since server start                                 |
+| `minio_cluster_replication_current_link_latency_ms`        | Current replication link latency in milliseconds                                                         |
+| `minio_cluster_replication_current_transfer_rate`          | Current replication transfer rate in bytes/sec                                                           |
+| `minio_cluster_replication_average_transfer_rate`          | Average replication transfer rate in bytes/sec                                                           |
+| `minio_cluster_replication_max_transfer_rate`              | Maximum replication transfer rate in bytes/sec seen since server start                                   |
+| `minio_cluster_replication_last_minute_queued_count`       | Total number of objects queued for replication in the last full minute                                   |
+| `minio_cluster_replication_last_minute_queued_bytes`       | Total number of bytes queued for replication in the last full minute                                     |
+| `minio_cluster_replication_average_queued_count`           | Average number of objects queued for replication since server start                                      |
+| `minio_cluster_replication_average_queued_bytes`           | Average number of bytes queued for replication since server start                                        |
+| `minio_cluster_replication_max_queued_bytes`               | Maximum number of bytes queued for replication seen since server start                                   |
+| `minio_cluster_replication_max_queued_count`               | Maximum number of objects queued for replication seen since server start                                 |
+| `minio_cluster_replication_recent_backlog_count`           | Total number of objects seen in replication backlog in the last 5 minutes                                |
+| `minio_cluster_replication_last_minute_failed_bytes`       | Total number of bytes failed at least once to replicate in the last full minute.                         |
+| `minio_cluster_replication_last_minute_failed_count`       | Total number of objects which failed replication in the last full minute.                                |
+| `minio_cluster_replication_last_hour_failed_bytes`         | (_Site Replication Only_) Total number of bytes failed at least once to replicate in the last full hour. |
+| `minio_cluster_replication_last_hour_failed_count`         | (_Site Replication Only_) Total number of objects which failed replication in the last full hour.        |
+| `minio_cluster_replication_total_failed_bytes`             | (_Site Replication Only_) Total number of bytes failed at least once to replicate since server start.    |
+| `minio_cluster_replication_total_failed_count`             | (_Site Replication Only_) Total number of objects which failed replication since server start.           |
+| `minio_cluster_replication_received_bytes`                 | (_Site Replication Only_) Total number of bytes replicated to this cluster from another source cluster.  |
+| `minio_cluster_replication_received_count`                 | (_Site Replication Only_) Total number of objects received by this cluster from another source cluster.  |
+| `minio_cluster_replication_sent_bytes`                     | (_Site Replication Only_) Total number of bytes replicated to the target cluster.                        |
+| `minio_cluster_replication_sent_count`                     | (_Site Replication Only_) Total number of objects replicated to the target cluster.                      |
+| `minio_cluster_replication_credential_errors`              | (_Site Replication Only_) Total number of replication credential errors since server start               |
 
 ## Healing Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_heal_objects_errors_total`             | Objects for which healing failed in current self healing run.                                                   |
-| `minio_heal_objects_heal_total`               | Objects healed in current self healing run.                                                                     |
-| `minio_heal_objects_total`                    | Objects scanned in current self healing run.                                                                    |
-| `minio_heal_time_last_activity_nano_seconds`  | Time elapsed (in nano seconds) since last self healing activity.                                                |
+| Name                                         | Description                                                      |
+|:---------------------------------------------|:-----------------------------------------------------------------|
+| `minio_heal_objects_errors_total`            | Objects for which healing failed in current self healing run.    |
+| `minio_heal_objects_heal_total`              | Objects healed in current self healing run.                      |
+| `minio_heal_objects_total`                   | Objects scanned in current self healing run.                     |
+| `minio_heal_time_last_activity_nano_seconds` | Time elapsed (in nano seconds) since last self healing activity. |
 
 ## Inter Node Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_inter_node_traffic_dial_avg_time`      | Average time of internodes TCP dial calls.                                                                      |
-| `minio_inter_node_traffic_dial_errors`        | Total number of internode TCP dial timeouts and errors.                                                         |
-| `minio_inter_node_traffic_errors_total`       | Total number of failed internode calls.                                                                         |
-| `minio_inter_node_traffic_received_bytes`     | Total number of bytes received from other peer nodes.                                                           |
-| `minio_inter_node_traffic_sent_bytes`         | Total number of bytes sent to the other peer nodes.                                                             |
+| Name                                      | Description                                             |
+|:------------------------------------------|:--------------------------------------------------------|
+| `minio_inter_node_traffic_dial_avg_time`  | Average time of internodes TCP dial calls.              |
+| `minio_inter_node_traffic_dial_errors`    | Total number of internode TCP dial timeouts and errors. |
+| `minio_inter_node_traffic_errors_total`   | Total number of failed internode calls.                 |
+| `minio_inter_node_traffic_received_bytes` | Total number of bytes received from other peer nodes.   |
+| `minio_inter_node_traffic_sent_bytes`     | Total number of bytes sent to the other peer nodes.     |
 
 ## Bucket Notification Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_notify_current_send_in_progress`       | Number of concurrent async Send calls active to all targets.                                                    |
-| `minio_notify_target_queue_length`            | Number of unsent notifications in queue for target.                                                             |
+| Name                                    | Description                                                  |
+|:----------------------------------------|:-------------------------------------------------------------|
+| `minio_notify_current_send_in_progress` | Number of concurrent async Send calls active to all targets. |
+| `minio_notify_target_queue_length`      | Number of unsent notifications in queue for target.          |
+|                                         |                                                              |
 
 ## S3 API Request Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_s3_requests_4xx_errors_total`          | Total number S3 requests with (4xx) errors.                                                                     |
-| `minio_s3_requests_5xx_errors_total`          | Total number S3 requests with (5xx) errors.                                                                     |
-| `minio_s3_requests_canceled_total`            | Total number S3 requests canceled by the client.                                                                |
-| `minio_s3_requests_errors_total`              | Total number S3 requests with (4xx and 5xx) errors.                                                             |
-| `minio_s3_requests_incoming_total`            | Volatile number of total incoming S3 requests.                                                                  |
-| `minio_s3_requests_inflight_total`            | Total number of S3 requests currently in flight.                                                                |
-| `minio_s3_requests_rejected_auth_total`       | Total number S3 requests rejected for auth failure.                                                             |
-| `minio_s3_requests_rejected_header_total`     | Total number S3 requests rejected for invalid header.                                                           |
-| `minio_s3_requests_rejected_invalid_total`    | Total number S3 invalid requests.                                                                               |
-| `minio_s3_requests_rejected_timestamp_total`  | Total number S3 requests rejected for invalid timestamp.                                                        |
-| `minio_s3_requests_total`                     | Total number S3 requests.                                                                                       |
-| `minio_s3_requests_waiting_total`             | Number of S3 requests in the waiting queue.                                                                     |
-| `minio_s3_requests_ttfb_seconds_distribution` | Distribution of the time to first byte across API calls.                                                        |
-| `minio_s3_traffic_received_bytes`             | Total number of s3 bytes received.                                                                              |
-| `minio_s3_traffic_sent_bytes`                 | Total number of s3 bytes sent.                                                                                  |
+| Name                                          | Description                                              |
+|:----------------------------------------------|:---------------------------------------------------------|
+| `minio_s3_requests_4xx_errors_total`          | Total number S3 requests with (4xx) errors.              |
+| `minio_s3_requests_5xx_errors_total`          | Total number S3 requests with (5xx) errors.              |
+| `minio_s3_requests_canceled_total`            | Total number S3 requests canceled by the client.         |
+| `minio_s3_requests_errors_total`              | Total number S3 requests with (4xx and 5xx) errors.      |
+| `minio_s3_requests_incoming_total`            | Volatile number of total incoming S3 requests.           |
+| `minio_s3_requests_inflight_total`            | Total number of S3 requests currently in flight.         |
+| `minio_s3_requests_rejected_auth_total`       | Total number S3 requests rejected for auth failure.      |
+| `minio_s3_requests_rejected_header_total`     | Total number S3 requests rejected for invalid header.    |
+| `minio_s3_requests_rejected_invalid_total`    | Total number S3 invalid requests.                        |
+| `minio_s3_requests_rejected_timestamp_total`  | Total number S3 requests rejected for invalid timestamp. |
+| `minio_s3_requests_total`                     | Total number S3 requests.                                |
+| `minio_s3_requests_waiting_total`             | Number of S3 requests in the waiting queue.              |
+| `minio_s3_requests_ttfb_seconds_distribution` | Distribution of the time to first byte across API calls. |
+| `minio_s3_traffic_received_bytes`             | Total number of s3 bytes received.                       |
+| `minio_s3_traffic_sent_bytes`                 | Total number of s3 bytes sent.                           |
 
 ## Software Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_software_commit_info`                  | Git commit hash for the MinIO release.                                                                          |
-| `minio_software_version_info`                 | MinIO Release tag for the server.                                                                               |
+| Name                          | Description                            |
+|:------------------------------|:---------------------------------------|
+| `minio_software_commit_info`  | Git commit hash for the MinIO release. |
+| `minio_software_version_info` | MinIO Release tag for the server.      |
 
 ## Drive Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_node_drive_free_bytes`                 | Total storage available on a drive.                                                                             |
-| `minio_node_drive_free_inodes`                | Total free inodes.                                                                                              |
-| `minio_node_drive_latency_us`                 | Average last minute latency in µs for drive API storage operations.                                             |
-| `minio_node_drive_offline_total`              | Total drives offline in this node.                                                                              |
-| `minio_node_drive_online_total`               | Total drives online in this node.                                                                               |
-| `minio_node_drive_total`                      | Total drives in this node.                                                                                      |
-| `minio_node_drive_total_bytes`                | Total storage on a drive.                                                                                       |
-| `minio_node_drive_used_bytes`                 | Total storage used on a drive.                                                                                  |
+| Name                             | Description                                                         |
+|:---------------------------------|:--------------------------------------------------------------------|
+| `minio_node_drive_free_bytes`    | Total storage available on a drive.                                 |
+| `minio_node_drive_free_inodes`   | Total free inodes.                                                  |
+| `minio_node_drive_latency_us`    | Average last minute latency in µs for drive API storage operations. |
+| `minio_node_drive_offline_total` | Total drives offline in this node.                                  |
+| `minio_node_drive_online_total`  | Total drives online in this node.                                   |
+| `minio_node_drive_total`         | Total drives in this node.                                          |
+| `minio_node_drive_total_bytes`   | Total storage on a drive.                                           |
+| `minio_node_drive_used_bytes`    | Total storage used on a drive.                                      |
 
 ## Identity and Access Management (IAM) Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_node_iam_last_sync_duration_millis`    | Last successful IAM data sync duration in milliseconds.                                                         |
-| `minio_node_iam_since_last_sync_millis`       | Time (in milliseconds) since last successful IAM data sync.                                                     |
-| `minio_node_iam_sync_failures`                | Number of failed IAM data syncs since server start.                                                             |
-| `minio_node_iam_sync_successes`               | Number of successful IAM data syncs since server start.                                                         |
+| Name                                       | Description                                                 |
+|:-------------------------------------------|:------------------------------------------------------------|
+| `minio_node_iam_last_sync_duration_millis` | Last successful IAM data sync duration in milliseconds.     |
+| `minio_node_iam_since_last_sync_millis`    | Time (in milliseconds) since last successful IAM data sync. |
+| `minio_node_iam_sync_failures`             | Number of failed IAM data syncs since server start.         |
+| `minio_node_iam_sync_successes`            | Number of successful IAM data syncs since server start.     |
 
 ## Information Lifecycle Management (ILM) Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_node_ilm_expiry_pending_tasks`         | Number of pending ILM expiry tasks in the queue.                                                                |
-| `minio_node_ilm_transition_active_tasks`      | Number of active ILM transition tasks.                                                                          |
-| `minio_node_ilm_transition_pending_tasks`     | Number of pending ILM transition tasks in the queue.                                                            |
-| `minio_node_ilm_versions_scanned`             | Total number of object versions checked for ilm actions since server start.                                     |
+| Name                                               | Description                                                                 |
+|:---------------------------------------------------|:----------------------------------------------------------------------------|
+| `minio_node_ilm_expiry_pending_tasks`              | Number of pending ILM expiry tasks in the queue.                            |
+| `minio_node_ilm_transition_active_tasks`           | Number of active ILM transition tasks.                                      |
+| `minio_node_ilm_transition_pending_tasks`          | Number of pending ILM transition tasks in the queue.                        |
+| `minio_node_ilm_transition_missed_immediate_tasks` | Number of missed immediate ILM transition tasks.                            |
+| `minio_node_ilm_versions_scanned`                  | Total number of object versions checked for ilm actions since server start. |
 
 ## System Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_node_file_descriptor_limit_total`      | Limit on total number of open file descriptors for the MinIO Server process.                                    |
-| `minio_node_file_descriptor_open_total`       | Total number of open file descriptors by the MinIO Server process.                                              |
-| `minio_node_go_routine_total`                 | Total number of go routines running.                                                                            |
-| `minio_node_io_rchar_bytes`                   | Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar.       |
-| `minio_node_io_read_bytes`                    | Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes.                  |
-| `minio_node_io_wchar_bytes`                   | Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar. |
-| `minio_node_io_write_bytes`                   | Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes.                |
-| `minio_node_process_cpu_total_seconds`        | Total user and system CPU time spent in seconds.                                                                |
-| `minio_node_process_resident_memory_bytes`    | Resident memory size in bytes.                                                                                  |
-| `minio_node_process_starttime_seconds`        | Start time for MinIO process per node, time in seconds since Unix epoc.                                         |
-| `minio_node_process_uptime_seconds`           | Uptime for MinIO process per node in seconds.                                                                   |
+| Name                                       | Description                                                                                                     |
+|:-------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| `minio_node_file_descriptor_limit_total`   | Limit on total number of open file descriptors for the MinIO Server process.                                    |
+| `minio_node_file_descriptor_open_total`    | Total number of open file descriptors by the MinIO Server process.                                              |
+| `minio_node_go_routine_total`              | Total number of go routines running.                                                                            |
+| `minio_node_io_rchar_bytes`                | Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar.       |
+| `minio_node_io_read_bytes`                 | Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes.                  |
+| `minio_node_io_wchar_bytes`                | Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar. |
+| `minio_node_io_write_bytes`                | Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes.                |
+| `minio_node_process_cpu_total_seconds`     | Total user and system CPU time spent in seconds.                                                                |
+| `minio_node_process_resident_memory_bytes` | Resident memory size in bytes.                                                                                  |
+| `minio_node_process_starttime_seconds`     | Start time for MinIO process per node, time in seconds since Unix epoc.                                         |
+| `minio_node_process_uptime_seconds`        | Uptime for MinIO process per node in seconds.                                                                   |
 
 ## Scanner Metrics
 
-| Name                                          | Description                                                                                                     |
-|:----------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| `minio_node_scanner_bucket_scans_finished`    | Total number of bucket scans finished since server start.                                                       |
-| `minio_node_scanner_bucket_scans_started`     | Total number of bucket scans started since server start.                                                        |
-| `minio_node_scanner_directories_scanned`      | Total number of directories scanned since server start.                                                         |
-| `minio_node_scanner_objects_scanned`          | Total number of unique objects scanned since server start.                                                      |
-| `minio_node_scanner_versions_scanned`         | Total number of object versions scanned since server start.                                                     |
-| `minio_node_syscall_read_total`               | Total read SysCalls to the kernel. /proc/[pid]/io syscr.                                                        |
-| `minio_node_syscall_write_total`              | Total write SysCalls to the kernel. /proc/[pid]/io syscw.                                                       |
-| `minio_usage_last_activity_nano_seconds`      | Time elapsed (in nano seconds) since last scan activity.                                                        |
+| Name                                       | Description                                                 |
+|:-------------------------------------------|:------------------------------------------------------------|
+| `minio_node_scanner_bucket_scans_finished` | Total number of bucket scans finished since server start.   |
+| `minio_node_scanner_bucket_scans_started`  | Total number of bucket scans started since server start.    |
+| `minio_node_scanner_directories_scanned`   | Total number of directories scanned since server start.     |
+| `minio_node_scanner_objects_scanned`       | Total number of unique objects scanned since server start.  |
+| `minio_node_scanner_versions_scanned`      | Total number of object versions scanned since server start. |
+| `minio_node_syscall_read_total`            | Total read SysCalls to the kernel. /proc/[pid]/io syscr.    |
+| `minio_node_syscall_write_total`           | Total write SysCalls to the kernel. /proc/[pid]/io syscw.   |
+| `minio_usage_last_activity_nano_seconds`   | Time elapsed (in nano seconds) since last scan activity.    |
 
 # Bucket Metrics
 
@@ -251,10 +242,10 @@ For deployments behind a load balancer, use the load balancer hostname instead o
 
 ## Distribution Metrics
 
-| Name                                              | Description                                                                     |
-|:--------------------------------------------------|:--------------------------------------------------------------------------------|
-| `minio_bucket_objects_size_distribution`          | Distribution of object sizes in the bucket, includes label for the bucket name. |
-| `minio_bucket_objects_version_distribution`       | Distribution of object sizes in a bucket, by number of versions                 |
+| Name                                        | Description                                                                     |
+|:--------------------------------------------|:--------------------------------------------------------------------------------|
+| `minio_bucket_objects_size_distribution`    | Distribution of object sizes in the bucket, includes label for the bucket name. |
+| `minio_bucket_objects_version_distribution` | Distribution of object sizes in a bucket, by number of versions                 |
 
 ## Replication Metrics
 
@@ -278,31 +269,31 @@ For deployments with [Site Replication](https://min.io/docs/minio/linux/operatio
 
 ## Traffic Metrics
 
-| Name                                              | Description                                                                     |
-|:--------------------------------------------------|:--------------------------------------------------------------------------------|
-| `minio_bucket_traffic_received_bytes`             | Total number of S3 bytes received for this bucket.                              |
-| `minio_bucket_traffic_sent_bytes`                 | Total number of S3 bytes sent for this bucket.                                  |
-
+| Name                                  | Description                                        |
+|:--------------------------------------|:---------------------------------------------------|
+| `minio_bucket_traffic_received_bytes` | Total number of S3 bytes received for this bucket. |
+| `minio_bucket_traffic_sent_bytes`     | Total number of S3 bytes sent for this bucket.     |
+	
 ## Usage Metrics
 
-| Name                                              | Description                                                                     |
-|:--------------------------------------------------|:--------------------------------------------------------------------------------|
-| `minio_bucket_usage_object_total`                 | Total number of objects.                                                        |
-| `minio_bucket_usage_version_total`                | Total number of versions (includes delete marker)                               |
-| `minio_bucket_usage_deletemarker_total`           | Total number of delete markers.                                                 |
-| `minio_bucket_usage_total_bytes`                  | Total bucket size in bytes.                                                     |
-| `minio_bucket_quota_total_bytes`                  | Total bucket quota size in bytes.                                               |
+| Name                                    | Description                                       |
+|:----------------------------------------|:--------------------------------------------------|
+| `minio_bucket_usage_object_total`       | Total number of objects.                          |
+| `minio_bucket_usage_version_total`      | Total number of versions (includes delete marker) |
+| `minio_bucket_usage_deletemarker_total` | Total number of delete markers.                   |
+| `minio_bucket_usage_total_bytes`        | Total bucket size in bytes.                       |
+| `minio_bucket_quota_total_bytes`        | Total bucket quota size in bytes.                 |
 
 ## Requests Metrics
 
-| Name                                              | Description                                                                     |
-|:--------------------------------------------------|:--------------------------------------------------------------------------------|
-| `minio_bucket_requests_4xx_errors_total`          | Total number of S3 requests with (4xx) errors on a bucket.                      |
-| `minio_bucket_requests_5xx_errors_total`          | Total number of S3 requests with (5xx) errors on a bucket.                      |
-| `minio_bucket_requests_inflight_total`            | Total number of S3 requests currently in flight on a bucket.                    |
-| `minio_bucket_requests_total`                     | Total number of S3 requests on a bucket.                                        |
-| `minio_bucket_requests_canceled_total`            | Total number S3 requests canceled by the client.                                |
-| `minio_bucket_requests_ttfb_seconds_distribution` | Distribution of time to first byte across API calls per bucket.                 |
+| Name                                              | Description                                                     |
+|:--------------------------------------------------|:----------------------------------------------------------------|
+| `minio_bucket_requests_4xx_errors_total`          | Total number of S3 requests with (4xx) errors on a bucket.      |
+| `minio_bucket_requests_5xx_errors_total`          | Total number of S3 requests with (5xx) errors on a bucket.      |
+| `minio_bucket_requests_inflight_total`            | Total number of S3 requests currently in flight on a bucket.    |
+| `minio_bucket_requests_total`                     | Total number of S3 requests on a bucket.                        |
+| `minio_bucket_requests_canceled_total`            | Total number S3 requests canceled by the client.                |
+| `minio_bucket_requests_ttfb_seconds_distribution` | Distribution of time to first byte across API calls per bucket. |
 
 # Resource Metrics
 
@@ -322,7 +313,7 @@ For deployments behind a load balancer, use the load balancer hostname instead o
 ## Drive Resource Metrics
 
 | Name                                 | Description                                              |
-| :----------------------------------- | :------------------------------------------------------- |
+|:-------------------------------------|:---------------------------------------------------------|
 | `minio_node_drive_total_bytes`       | Total bytes on a drive.                                  |
 | `minio_node_drive_used_bytes`        | Used bytes on a drive.                                   |
 | `minio_node_drive_total_inodes`      | Total inodes on a drive.                                 |
@@ -338,7 +329,7 @@ For deployments behind a load balancer, use the load balancer hostname instead o
 ## Network Interface Metrics
 
 | Name                          | Description                                   |
-| :---------------------------- | :-------------------------------------------- |
+|:------------------------------|:----------------------------------------------|
 | `minio_node_if_rx_bytes`      | Bytes received on the interface in 60s.       |
 | `minio_node_if_rx_bytes_avg`  | Bytes received on the interface in 60s (avg). |
 | `minio_node_if_rx_bytes_max`  | Bytes received on the interface in 60s (max). |
@@ -354,58 +345,70 @@ For deployments behind a load balancer, use the load balancer hostname instead o
 
 ## CPU Metrics
 
-| Name                            | Description                   |
-| :------------------------------ | :---------------------------- |
-| `minio_node_cpu_avg_user`       | CPU user time.                |
-| `minio_node_cpu_avg_user_avg`   | CPU user time (avg).          |
-| `minio_node_cpu_avg_user_max`   | CPU user time (max).          |
-| `minio_node_cpu_avg_system`     | CPU system time.              |
-| `minio_node_cpu_avg_system_avg` | CPU system time (avg).        |
-| `minio_node_cpu_avg_system_max` | CPU system time (max).        |
-| `minio_node_cpu_avg_idle`       | CPU idle time.                |
-| `minio_node_cpu_avg_idle_avg`   | CPU idle time (avg).          |
-| `minio_node_cpu_avg_idle_max`   | CPU idle time (max).          |
-| `minio_node_cpu_avg_iowait`     | CPU ioWait time.              |
-| `minio_node_cpu_avg_iowait_avg` | CPU ioWait time (avg).        |
-| `minio_node_cpu_avg_iowait_max` | CPU ioWait time (max).        |
-| `minio_node_cpu_avg_nice`       | CPU nice time.                |
-| `minio_node_cpu_avg_nice_avg`   | CPU nice time (avg).          |
-| `minio_node_cpu_avg_nice_max`   | CPU nice time (max).          |
-| `minio_node_cpu_avg_steal`      | CPU steam time.               |
-| `minio_node_cpu_avg_steal_avg`  | CPU steam time (avg).         |
-| `minio_node_cpu_avg_steal_max`  | CPU steam time (max).         |
-| `minio_node_cpu_avg_load1`      | CPU load average 1min.        |
-| `minio_node_cpu_avg_load1_avg`  | CPU load average 1min (avg).  |
-| `minio_node_cpu_avg_load1_max`  | CPU load average 1min (max).  |
-| `minio_node_cpu_avg_load5`      | CPU load average 5min.        |
-| `minio_node_cpu_avg_load5_avg`  | CPU load average 5min (avg).  |
-| `minio_node_cpu_avg_load5_max`  | CPU load average 5min (max).  |
-| `minio_node_cpu_avg_load15`     | CPU load average 15min.       |
-| `minio_node_cpu_avg_load15_avg` | CPU load average 15min (avg). |
-| `minio_node_cpu_avg_load15_max` | CPU load average 15min (max). |
+| Name                                 | Description                                |
+|:-------------------------------------|:-------------------------------------------|
+| `minio_node_cpu_avg_user`            | CPU user time.                             |
+| `minio_node_cpu_avg_user_avg`        | CPU user time (avg).                       |
+| `minio_node_cpu_avg_user_max`        | CPU user time (max).                       |
+| `minio_node_cpu_avg_system`          | CPU system time.                           |
+| `minio_node_cpu_avg_system_avg`      | CPU system time (avg).                     |
+| `minio_node_cpu_avg_system_max`      | CPU system time (max).                     |
+| `minio_node_cpu_avg_idle`            | CPU idle time.                             |
+| `minio_node_cpu_avg_idle_avg`        | CPU idle time (avg).                       |
+| `minio_node_cpu_avg_idle_max`        | CPU idle time (max).                       |
+| `minio_node_cpu_avg_iowait`          | CPU ioWait time.                           |
+| `minio_node_cpu_avg_iowait_avg`      | CPU ioWait time (avg).                     |
+| `minio_node_cpu_avg_iowait_max`      | CPU ioWait time (max).                     |
+| `minio_node_cpu_avg_nice`            | CPU nice time.                             |
+| `minio_node_cpu_avg_nice_avg`        | CPU nice time (avg).                       |
+| `minio_node_cpu_avg_nice_max`        | CPU nice time (max).                       |
+| `minio_node_cpu_avg_steal`           | CPU steam time.                            |
+| `minio_node_cpu_avg_steal_avg`       | CPU steam time (avg).                      |
+| `minio_node_cpu_avg_steal_max`       | CPU steam time (max).                      |
+| `minio_node_cpu_avg_load1`           | CPU load average 1min.                     |
+| `minio_node_cpu_avg_load1_avg`       | CPU load average 1min (avg).               |
+| `minio_node_cpu_avg_load1_max`       | CPU load average 1min (max).               |
+| `minio_node_cpu_avg_load1_perc`      | CPU load average 1min (percentage).        |
+| `minio_node_cpu_avg_load1_perc_avg`  | CPU load average 1min (percentage) (avg).  |
+| `minio_node_cpu_avg_load1_perc_max`  | CPU load average 1min (percentage) (max).  |
+| `minio_node_cpu_avg_load5`           | CPU load average 5min.                     |
+| `minio_node_cpu_avg_load5_avg`       | CPU load average 5min (avg).               |
+| `minio_node_cpu_avg_load5_max`       | CPU load average 5min (max).               |
+| `minio_node_cpu_avg_load5_perc`      | CPU load average 5min (percentage).        |
+| `minio_node_cpu_avg_load5_perc_avg`  | CPU load average 5min (percentage) (avg).  |
+| `minio_node_cpu_avg_load5_perc_max`  | CPU load average 5min (percentage) (max).  |
+| `minio_node_cpu_avg_load15`          | CPU load average 15min.                    |
+| `minio_node_cpu_avg_load15_avg`      | CPU load average 15min (avg).              |
+| `minio_node_cpu_avg_load15_max`      | CPU load average 15min (max).              |
+| `minio_node_cpu_avg_load15_perc`     | CPU load average 15min (percentage).       |
+| `minio_node_cpu_avg_load15_perc_avg` | CPU load average 15min (percentage) (avg). |
+| `minio_node_cpu_avg_load15_perc_max` | CPU load average 15min (percentage) (max). |
 
 ## Memory Metrics
 
-| Name                           | Description                         |
-| :----------------------------- | :---------------------------------- |
-| `minio_node_mem_available`     | Available memory on the node.       |
-| `minio_node_mem_available_avg` | Available memory on the node (avg). |
-| `minio_node_mem_available_max` | Available memory on the node (max). |
-| `minio_node_mem_buffers`       | Buffers memory on the node.         |
-| `minio_node_mem_buffers_avg`   | Buffers memory on the node (avg).   |
-| `minio_node_mem_buffers_max`   | Buffers memory on the node (max).   |
-| `minio_node_mem_cache`         | Cache memory on the node.           |
-| `minio_node_mem_cache_avg`     | Cache memory on the node (avg).     |
-| `minio_node_mem_cache_max`     | Cache memory on the node (max).     |
-| `minio_node_mem_free`          | Free memory on the node.            |
-| `minio_node_mem_free_avg`      | Free memory on the node (avg).      |
-| `minio_node_mem_free_max`      | Free memory on the node (max).      |
-| `minio_node_mem_shared`        | Shared memory on the node.          |
-| `minio_node_mem_shared_avg`    | Shared memory on the node (avg).    |
-| `minio_node_mem_shared_max`    | Shared memory on the node (max).    |
-| `minio_node_mem_total`         | Total memory on the node.           |
-| `minio_node_mem_total_avg`     | Total memory on the node (avg).     |
-| `minio_node_mem_total_max`     | Total memory on the node (max).     |
-| `minio_node_mem_used`          | Used memory on the node.            |
-| `minio_node_mem_used_avg`      | Used memory on the node (avg).      |
-| `minio_node_mem_used_max`      | Used memory on the node (max).      |
+| Name                           | Description                               |
+|:-------------------------------|:------------------------------------------|
+| `minio_node_mem_available`     | Available memory on the node.             |
+| `minio_node_mem_available_avg` | Available memory on the node (avg).       |
+| `minio_node_mem_available_max` | Available memory on the node (max).       |
+| `minio_node_mem_buffers`       | Buffers memory on the node.               |
+| `minio_node_mem_buffers_avg`   | Buffers memory on the node (avg).         |
+| `minio_node_mem_buffers_max`   | Buffers memory on the node (max).         |
+| `minio_node_mem_cache`         | Cache memory on the node.                 |
+| `minio_node_mem_cache_avg`     | Cache memory on the node (avg).           |
+| `minio_node_mem_cache_max`     | Cache memory on the node (max).           |
+| `minio_node_mem_free`          | Free memory on the node.                  |
+| `minio_node_mem_free_avg`      | Free memory on the node (avg).            |
+| `minio_node_mem_free_max`      | Free memory on the node (max).            |
+| `minio_node_mem_shared`        | Shared memory on the node.                |
+| `minio_node_mem_shared_avg`    | Shared memory on the node (avg).          |
+| `minio_node_mem_shared_max`    | Shared memory on the node (max).          |
+| `minio_node_mem_total`         | Total memory on the node.                 |
+| `minio_node_mem_total_avg`     | Total memory on the node (avg).           |
+| `minio_node_mem_total_max`     | Total memory on the node (max).           |
+| `minio_node_mem_used`          | Used memory on the node.                  |
+| `minio_node_mem_used_avg`      | Used memory on the node (avg).            |
+| `minio_node_mem_used_max`      | Used memory on the node (max).            |
+| `minio_node_mem_used_perc`     | Used memory percentage on the node.       |
+| `minio_node_mem_used_perc_avg` | Used memory percentage on the node (avg). |
+| `minio_node_mem_used_perc_max` | Used memory percentage on the node (max). |

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2023-11-20T22-40-07Z
+  image: quay.io/minio/minio:RELEASE.2023-12-09T18-17-51Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/run-multi-site-ldap.sh

@@ -242,6 +242,19 @@ if [ "${expected_checksum}" != "${actual_checksum}" ]; then
 fi
 rm ./lrgfile
 
+./mc rm -r --versions --force minio1/newbucket/lrgfile
+if [ $? -ne 0 ]; then
+	echo "expected object to be present, exiting.."
+	exit_1
+fi
+
+sleep 5
+./mc stat minio1/newbucket/lrgfile
+if [ $? -eq 0 ]; then
+	echo "expected object to be deleted permanently after replication, exiting.."
+	exit_1
+fi
+
 vID=$(./mc stat minio2/newbucket/README.md --json | jq .versionID)
 if [ $? -ne 0 ]; then
 	echo "expecting object to be present. exiting.."

docs/site-replication/run-multi-site-minio-idp.sh

@@ -229,6 +229,19 @@ if [ "${expected_checksum}" != "${actual_checksum}" ]; then
 fi
 rm ./lrgfile
 
+./mc rm -r --versions --force minio1/newbucket/lrgfile
+if [ $? -ne 0 ]; then
+	echo "expected object to be present, exiting.."
+	exit_1
+fi
+
+sleep 5
+./mc stat minio1/newbucket/lrgfile
+if [ $? -eq 0 ]; then
+	echo "expected object to be deleted permanently after replication, exiting.."
+	exit_1
+fi
+
 vID=$(./mc stat minio2/newbucket/README.md --json | jq .versionID)
 if [ $? -ne 0 ]; then
 	echo "expecting object to be present. exiting.."
@@ -240,7 +253,11 @@ if [ $? -ne 0 ]; then
 	exit_1
 fi
 sleep 5
-
+val=$(./mc tag list minio1/newbucket/README.md --version-id "${vID}" --json | jq -r .tagset.key)
+if [ "${val}" != "val" ]; then
+	echo "expected bucket tag to have replicated, exiting..."
+	exit_1
+fi
 ./mc tag remove --version-id "${vID}" minio2/newbucket/README.md
 if [ $? -ne 0 ]; then
 	echo "expecting tag removal to be successful. exiting.."

docs/site-replication/run-multi-site-oidc.sh

@@ -229,6 +229,19 @@ if [ "${expected_checksum}" != "${actual_checksum}" ]; then
 fi
 rm ./lrgfile
 
+./mc rm -r --versions --force minio1/newbucket/lrgfile
+if [ $? -ne 0 ]; then
+	echo "expected object to be present, exiting.."
+	exit_1
+fi
+
+sleep 5
+./mc stat minio1/newbucket/lrgfile
+if [ $? -eq 0 ]; then
+	echo "expected object to be deleted permanently after replication, exiting.."
+	exit_1
+fi
+
 ./mc mb --with-lock minio3/newbucket-olock
 sleep 5
 

go.mod

@@ -1,10 +1,12 @@
 module github.com/minio/minio
 
-go 1.21
+go 1.19
 
 require (
 	cloud.google.com/go/storage v1.35.1
 	github.com/Azure/azure-storage-blob-go v0.15.0
+	github.com/Azure/go-autorest/autorest v0.11.29
+	github.com/Azure/go-autorest/autorest/adal v0.9.22
 	github.com/IBM/sarama v1.42.1
 	github.com/alecthomas/participle v0.7.1
 	github.com/bcicen/jstream v1.0.1
@@ -43,17 +45,16 @@ require (
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/miekg/dns v1.1.57
 	github.com/minio/cli v1.24.2
-	github.com/minio/console v0.41.0
+	github.com/minio/console v0.42.1
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dnscache v0.1.1
 	github.com/minio/dperf v0.5.2
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes-go v0.2.0
-	github.com/minio/madmin-go/v3 v3.0.35-0.20231130082526-199918d0ff20
+	github.com/minio/madmin-go/v3 v3.0.37-0.20231211192618-d20cff0b11d9
 	github.com/minio/minio-go/v7 v7.0.65-0.20231122233251-1f7dd6b7e3e1
 	github.com/minio/mux v1.9.0
-	github.com/minio/pkg v1.7.5
-	github.com/minio/pkg/v2 v2.0.4
+	github.com/minio/pkg/v2 v2.0.5-0.20231205011044-ec6b8d58893a
 	github.com/minio/selfupdate v0.6.0
 	github.com/minio/sha256-simd v1.0.1
 	github.com/minio/simdjson-go v0.4.5
@@ -106,6 +107,10 @@ require (
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.5 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
@@ -172,7 +177,7 @@ require (
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
-	github.com/lestrrat-go/jwx v1.2.26 // indirect
+	github.com/lestrrat-go/jwx v1.2.27 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
@@ -188,6 +193,7 @@ require (
 	github.com/minio/filepath v1.0.0 // indirect
 	github.com/minio/mc v0.0.0-20231127112613-5e6ae2172e25 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/pkg v1.7.5 // indirect
 	github.com/minio/websocket v1.6.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum

@@ -21,12 +21,15 @@ github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9B
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw=
+github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc=
 github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
+github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
 github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
@@ -148,7 +151,6 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
-github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fraugster/parquet-go v0.12.0 h1:1slnC5y2VWEOUSlzbeXatM0BvSWcLUDsR/EcZsXXCZc=
@@ -216,6 +218,7 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -259,7 +262,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
-github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/pprof v0.0.0-20231127191134-f3a68a39ae15 h1:t2sLhFuGXwoomaKLTuoxFfFqqlG1Gp2DpsupXq3UvZ0=
 github.com/google/pprof v0.0.0-20231127191134-f3a68a39ae15/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
@@ -374,15 +376,14 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
-github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
 github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
 github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
 github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
 github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
 github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
-github.com/lestrrat-go/jwx v1.2.26 h1:4iFo8FPRZGDYe1t19mQP0zTRqA7n8HnJ5lkIiDvJcB0=
-github.com/lestrrat-go/jwx v1.2.26/go.mod h1:MaiCdGbn3/cckbOFSCluJlJMmp9dmZm5hDuIkx8ftpQ=
+github.com/lestrrat-go/jwx v1.2.27 h1:cvnTnda/YzdyFuWdEAMkI6BsLtItSrASEVCI3C/IUEQ=
+github.com/lestrrat-go/jwx v1.2.27/go.mod h1:Stob9LjSqR3lOmNdxF0/TvZo60V3hUGv8Fr7Bwzla3k=
 github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
 github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
@@ -435,8 +436,8 @@ github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
 github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.6 h1:m7TUvpvt0u7FBmVIEQNIa0T4NBQlxrcMBp4wJKsg2Ik=
 github.com/minio/colorjson v1.0.6/go.mod h1:LUXwS5ZGNb6Eh9f+t+3uJiowD3XsIWtsvTriUBeqgYs=
-github.com/minio/console v0.41.0 h1:NjvBij5Hg4GLkO/iAUfZ4imATA/rKNtgVhnn3sEuKDo=
-github.com/minio/console v0.41.0/go.mod h1:LTDngEa3Z/s9+2oUb3eBtaVsS/vQFuWTH9d8Z2Pe1mo=
+github.com/minio/console v0.42.1 h1:b56c172tjhbBaFU0GRjxVLxxq5nJZlRVOxBFNZpjCzo=
+github.com/minio/console v0.42.1/go.mod h1:UkIrq7HsuOy36il+He1jkqBWrhRUGzeET/8A0XC9bB4=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dnscache v0.1.1 h1:AMYLqomzskpORiUA1ciN9k7bZT1oB3YZN4cEIi88W5o=
@@ -449,8 +450,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/kes-go v0.2.0 h1:HA33arq9s3MErbsj3PAXFVfFo4U4yw7lTKQ5kWFrpCA=
 github.com/minio/kes-go v0.2.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo=
-github.com/minio/madmin-go/v3 v3.0.35-0.20231130082526-199918d0ff20 h1:5kfjAypPN18QOOQaZjR3jfGzXyIwzLdKMS7d/cPY3Wc=
-github.com/minio/madmin-go/v3 v3.0.35-0.20231130082526-199918d0ff20/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8=
+github.com/minio/madmin-go/v3 v3.0.37-0.20231211192618-d20cff0b11d9 h1:Rpz09w+Y9Bcq3MvNbxA/IFynLjVm6L62o5P7oMwNMWc=
+github.com/minio/madmin-go/v3 v3.0.37-0.20231211192618-d20cff0b11d9/go.mod h1:4QN2NftLSV7MdlT50dkrenOMmNVHluxTvlqJou3hte8=
 github.com/minio/mc v0.0.0-20231127112613-5e6ae2172e25 h1:8jT9Tz4opgrX6mnyFWW+TQ90AnrJqJ0mzeFXUWDHNGo=
 github.com/minio/mc v0.0.0-20231127112613-5e6ae2172e25/go.mod h1:8kat72LmpzZ2/xykDcq64tcRRJkkWo1Kd/Z5coC6t0w=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
@@ -462,8 +463,8 @@ github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA=
 github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ=
 github.com/minio/pkg v1.7.5 h1:UOUJjewE5zoaDPlCMJtNx/swc1jT1ZR+IajT7hrLd44=
 github.com/minio/pkg v1.7.5/go.mod h1:mEfGMTm5Z0b5EGxKNuPwyb5A2d+CC/VlUyRj6RJtIwo=
-github.com/minio/pkg/v2 v2.0.4 h1:vHQOqxD7AdsJq3X9GylO0PO6WCpcK7aePsVfk0oRYh0=
-github.com/minio/pkg/v2 v2.0.4/go.mod h1:ya1jPY/rtVp3HarxwDQJkclR8Rd2QxwxA9kwS436ovs=
+github.com/minio/pkg/v2 v2.0.5-0.20231205011044-ec6b8d58893a h1:xQwC1vKI7rVnwMufSXaNF1Ns7LycIhvftiH1s3yFe58=
+github.com/minio/pkg/v2 v2.0.5-0.20231205011044-ec6b8d58893a/go.mod h1:ya1jPY/rtVp3HarxwDQJkclR8Rd2QxwxA9kwS436ovs=
 github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
 github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -558,7 +559,6 @@ github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo=
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
@@ -607,7 +607,6 @@ github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZe
 github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
-github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
 github.com/rs/cors v1.10.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
@@ -692,7 +691,6 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
 github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
-github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
@@ -735,10 +733,10 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
 golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=

helm/minio/templates/_helper_create_user.txt

@@ -97,7 +97,7 @@ echo {{ tpl .accessKey $global }} > $MINIO_ACCESSKEY_SECRETKEY_TMP
 {{- if .existingSecret }}
 cat /config/secrets/{{ tpl .existingSecret $global }}/{{ tpl .existingSecretKey $global }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP
 # Add a new line if it doesn't exist
-sed -i '$a\' $MINIO_ACCESSKEY_SECRETKEY_TMP
+echo >> $MINIO_ACCESSKEY_SECRETKEY_TMP
 createUser {{ .policy }}
 {{ else }}
 echo {{ .secretKey }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP

internal/config/api/api.go

@@ -31,13 +31,15 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax                 = "requests_max"
-	apiRequestsDeadline            = "requests_deadline"
-	apiClusterDeadline             = "cluster_deadline"
-	apiCorsAllowOrigin             = "cors_allow_origin"
-	apiRemoteTransportDeadline     = "remote_transport_deadline"
-	apiListQuorum                  = "list_quorum"
-	apiReplicationPriority         = "replication_priority"
+	apiRequestsMax             = "requests_max"
+	apiRequestsDeadline        = "requests_deadline"
+	apiClusterDeadline         = "cluster_deadline"
+	apiCorsAllowOrigin         = "cors_allow_origin"
+	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiReplicationPriority     = "replication_priority"
+	apiReplicationMaxWorkers   = "replication_max_workers"
+
 	apiTransitionWorkers           = "transition_workers"
 	apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
 	apiStaleUploadsExpiry          = "stale_uploads_expiry"
@@ -48,16 +50,16 @@ const (
 	apiRootAccess                  = "root_access"
 	apiSyncEvents                  = "sync_events"
 
-	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
-	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
-	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
-	EnvAPITransitionWorkers       = "MINIO_API_TRANSITION_WORKERS"
-	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
-	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS" // default config.EnableOn
-	EnvAPIReplicationPriority     = "MINIO_API_REPLICATION_PRIORITY"
-
+	EnvAPIRequestsMax                 = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline            = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIClusterDeadline             = "MINIO_API_CLUSTER_DEADLINE"
+	EnvAPICorsAllowOrigin             = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRemoteTransportDeadline     = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
+	EnvAPITransitionWorkers           = "MINIO_API_TRANSITION_WORKERS"
+	EnvAPIListQuorum                  = "MINIO_API_LIST_QUORUM"
+	EnvAPISecureCiphers               = "MINIO_API_SECURE_CIPHERS" // default config.EnableOn
+	EnvAPIReplicationPriority         = "MINIO_API_REPLICATION_PRIORITY"
+	EnvAPIReplicationMaxWorkers       = "MINIO_API_REPLICATION_MAX_WORKERS"
 	EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL"
 	EnvAPIStaleUploadsExpiry          = "MINIO_API_STALE_UPLOADS_EXPIRY"
 	EnvAPIDeleteCleanupInterval       = "MINIO_API_DELETE_CLEANUP_INTERVAL"
@@ -107,6 +109,10 @@ var (
 			Key:   apiReplicationPriority,
 			Value: "auto",
 		},
+		config.KV{
+			Key:   apiReplicationMaxWorkers,
+			Value: "500",
+		},
 		config.KV{
 			Key:   apiTransitionWorkers,
 			Value: "100",
@@ -156,6 +162,7 @@ type Config struct {
 	RemoteTransportDeadline     time.Duration `json:"remote_transport_deadline"`
 	ListQuorum                  string        `json:"list_quorum"`
 	ReplicationPriority         string        `json:"replication_priority"`
+	ReplicationMaxWorkers       int           `json:"replication_max_workers"`
 	TransitionWorkers           int           `json:"transition_workers"`
 	StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
 	StaleUploadsExpiry          time.Duration `json:"stale_uploads_expiry"`
@@ -259,7 +266,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, fmt.Errorf("invalid value %v for replication_priority", replicationPriority)
 	}
 	cfg.ReplicationPriority = replicationPriority
+	replicationMaxWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationMaxWorkers, kvs.GetWithDefault(apiReplicationMaxWorkers, DefaultKVS)))
+	if err != nil {
+		return cfg, err
+	}
 
+	if replicationMaxWorkers <= 0 || replicationMaxWorkers > 500 {
+		return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Number of replication workers should be between 1 and 500")
+	}
+	cfg.ReplicationMaxWorkers = replicationMaxWorkers
 	transitionWorkers, err := strconv.Atoi(env.Get(EnvAPITransitionWorkers, kvs.GetWithDefault(apiTransitionWorkers, DefaultKVS)))
 	if err != nil {
 		return cfg, err

internal/config/api/help.go

@@ -68,6 +68,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         apiReplicationMaxWorkers,
+			Description: `set the maximum number of replication workers` + defaultHelpPostfix(apiReplicationMaxWorkers),
+			Optional:    true,
+			Type:        "number",
+		},
 		config.HelpKV{
 			Key:         apiTransitionWorkers,
 			Description: `set the number of transition workers` + defaultHelpPostfix(apiTransitionWorkers),

internal/config/batch/batch.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
 )
 
 // Batch job environment variables

internal/config/config.go

@@ -1311,9 +1311,8 @@ func (c Config) getTargetKVS(subSys, target string, redactSecrets bool) KVS {
 			// clonedKV := kv
 			// clonedKV.Value = redactedSecret
 			// resultKVS = append(resultKVS, clonedKV)
-		} else {
-			resultKVS = append(resultKVS, kv)
 		}
+		resultKVS = append(resultKVS, kv)
 	}
 
 	return resultKVS

internal/config/notify/parse.go

@@ -249,7 +249,7 @@ func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, t
 
 // FetchEnabledTargets - Returns a set of configured TargetList
 func FetchEnabledTargets(ctx context.Context, cfg config.Config, transport *http.Transport) (_ *event.TargetList, err error) {
-	targetList := event.NewTargetList()
+	targetList := event.NewTargetList(ctx)
 	for _, subSys := range config.NotifySubSystems.ToSlice() {
 		targets, err := fetchSubSysTargets(ctx, cfg, subSys, transport)
 		if err != nil {

internal/config/server.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+// Opts holds MinIO configuration options
+type Opts struct {
+	FTP struct {
+		Address          string `yaml:"address"`
+		PassivePortRange string `yaml:"passive-port-range"`
+	} `yaml:"ftp"`
+	SFTP struct {
+		Address       string `yaml:"address"`
+		SSHPrivateKey string `yaml:"ssh-private-key"`
+	} `yaml:"sftp"`
+}
+
+// ServerConfig represents a MinIO configuration file
+type ServerConfig struct {
+	Version     string     `yaml:"version"`
+	RootUser    string     `yaml:"rootUser"`
+	RootPwd     string     `yaml:"rootPassword"`
+	Addr        string     `yaml:"address"`
+	ConsoleAddr string     `yaml:"console-address"`
+	CertsDir    string     `yaml:"certs-dir"`
+	Pools       [][]string `yaml:"pools"`
+	Options     Opts       `yaml:"options"`
+}

internal/event/config_test.go

@@ -18,6 +18,7 @@
 package event
 
 import (
+	"context"
 	"encoding/xml"
 	"reflect"
 	"strings"
@@ -251,9 +252,9 @@ func TestQueueValidate(t *testing.T) {
 		panic(err)
 	}
 
-	targetList1 := NewTargetList()
+	targetList1 := NewTargetList(context.Background())
 
-	targetList2 := NewTargetList()
+	targetList2 := NewTargetList(context.Background())
 	if err := targetList2.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil {
 		panic(err)
 	}
@@ -595,9 +596,9 @@ func TestConfigValidate(t *testing.T) {
 		panic(err)
 	}
 
-	targetList1 := NewTargetList()
+	targetList1 := NewTargetList(context.Background())
 
-	targetList2 := NewTargetList()
+	targetList2 := NewTargetList(context.Background())
 	if err := targetList2.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil {
 		panic(err)
 	}
@@ -927,9 +928,9 @@ func TestParseConfig(t *testing.T) {
 </NotificationConfiguration>
 `)
 
-	targetList1 := NewTargetList()
+	targetList1 := NewTargetList(context.Background())
 
-	targetList2 := NewTargetList()
+	targetList2 := NewTargetList(context.Background())
 	if err := targetList2.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil {
 		panic(err)
 	}

internal/event/targetlist.go

@@ -18,17 +18,21 @@
 package event
 
 import (
+	"context"
 	"fmt"
+	"runtime"
 	"strings"
 	"sync"
 	"sync/atomic"
 
+	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/store"
+	"github.com/minio/pkg/v2/workers"
 )
 
 const (
 	// The maximum allowed number of concurrent Send() calls to all configured notifications targets
-	maxConcurrentTargetSendCalls = 20000
+	maxConcurrentAsyncSend = 50000
 )
 
 // Target - event target interface
@@ -49,9 +53,11 @@ type TargetStore interface {
 // TargetStats is a collection of stats for multiple targets.
 type TargetStats struct {
 	// CurrentSendCalls is the number of concurrent async Send calls to all targets
-	CurrentSendCalls int64
-	TotalEvents      int64
-	EventsSkipped    int64
+	CurrentSendCalls   int64
+	TotalEvents        int64
+	EventsSkipped      int64
+	CurrentQueuedCalls int64
+	EventsErrorsTotal  int64
 
 	TargetStats map[string]TargetStat
 }
@@ -65,12 +71,20 @@ type TargetStat struct {
 // TargetList - holds list of targets indexed by target ID.
 type TargetList struct {
 	// The number of concurrent async Send calls to all targets
-	currentSendCalls int64
-	totalEvents      int64
-	eventsSkipped    int64
+	currentSendCalls  atomic.Int64
+	totalEvents       atomic.Int64
+	eventsSkipped     atomic.Int64
+	eventsErrorsTotal atomic.Int64
 
 	sync.RWMutex
 	targets map[TargetID]Target
+	queue   chan asyncEvent
+	ctx     context.Context
+}
+
+type asyncEvent struct {
+	ev        Event
+	targetSet TargetIDSet
 }
 
 // Add - adds unique target to target list.
@@ -150,6 +164,14 @@ func (list *TargetList) List() []TargetID {
 	return keys
 }
 
+func (list *TargetList) get(id TargetID) (Target, bool) {
+	list.RLock()
+	defer list.RUnlock()
+
+	target, ok := list.targets[id]
+	return target, ok
+}
+
 // TargetMap - returns available targets.
 func (list *TargetList) TargetMap() map[TargetID]Target {
 	list.RLock()
@@ -163,46 +185,58 @@ func (list *TargetList) TargetMap() map[TargetID]Target {
 }
 
 // Send - sends events to targets identified by target IDs.
-func (list *TargetList) Send(event Event, targetIDset TargetIDSet, resCh chan<- TargetIDResult, synchronous bool) {
-	if atomic.LoadInt64(&list.currentSendCalls) > maxConcurrentTargetSendCalls {
-		atomic.AddInt64(&list.eventsSkipped, 1)
-		err := fmt.Errorf("concurrent target notifications exceeded %d", maxConcurrentTargetSendCalls)
-		for id := range targetIDset {
-			resCh <- TargetIDResult{ID: id, Err: err}
-		}
-		return
+func (list *TargetList) Send(event Event, targetIDset TargetIDSet, sync bool) {
+	if sync {
+		list.sendSync(event, targetIDset)
+	} else {
+		list.sendAsync(event, targetIDset)
 	}
-	if synchronous {
-		list.send(event, targetIDset, resCh)
-		return
-	}
-	go list.send(event, targetIDset, resCh)
 }
 
-func (list *TargetList) send(event Event, targetIDset TargetIDSet, resCh chan<- TargetIDResult) {
+func (list *TargetList) sendSync(event Event, targetIDset TargetIDSet) {
 	var wg sync.WaitGroup
 	for id := range targetIDset {
-		list.RLock()
-		target, ok := list.targets[id]
-		list.RUnlock()
-		if ok {
-			wg.Add(1)
-			go func(id TargetID, target Target) {
-				atomic.AddInt64(&list.currentSendCalls, 1)
-				defer atomic.AddInt64(&list.currentSendCalls, -1)
-				defer wg.Done()
-				tgtRes := TargetIDResult{ID: id}
-				if err := target.Save(event); err != nil {
-					tgtRes.Err = err
-				}
-				resCh <- tgtRes
-			}(id, target)
-		} else {
-			resCh <- TargetIDResult{ID: id}
+		target, ok := list.get(id)
+		if !ok {
+			continue
 		}
+		wg.Add(1)
+		go func(id TargetID, target Target) {
+			list.currentSendCalls.Add(1)
+			defer list.currentSendCalls.Add(-1)
+			defer wg.Done()
+
+			if err := target.Save(event); err != nil {
+				list.eventsErrorsTotal.Add(1)
+				reqInfo := &logger.ReqInfo{}
+				reqInfo.AppendTags("targetID", id.String())
+				logger.LogOnceIf(logger.SetReqInfo(context.Background(), reqInfo), err, id.String())
+			}
+		}(id, target)
 	}
 	wg.Wait()
-	atomic.AddInt64(&list.totalEvents, 1)
+	list.totalEvents.Add(1)
+}
+
+func (list *TargetList) sendAsync(event Event, targetIDset TargetIDSet) {
+	select {
+	case list.queue <- asyncEvent{
+		ev:        event,
+		targetSet: targetIDset.Clone(),
+	}:
+	case <-list.ctx.Done():
+		list.eventsSkipped.Add(int64(len(list.queue)))
+		return
+	default:
+		list.eventsSkipped.Add(1)
+		err := fmt.Errorf("concurrent target notifications exceeded %d, notification endpoint is too slow to accept events on incoming requests", maxConcurrentAsyncSend)
+		for id := range targetIDset {
+			reqInfo := &logger.ReqInfo{}
+			reqInfo.AppendTags("targetID", id.String())
+			logger.LogOnceIf(logger.SetReqInfo(context.Background(), reqInfo), err, id.String())
+		}
+		return
+	}
 }
 
 // Stats returns stats for targets.
@@ -211,9 +245,11 @@ func (list *TargetList) Stats() TargetStats {
 	if list == nil {
 		return t
 	}
-	t.CurrentSendCalls = atomic.LoadInt64(&list.currentSendCalls)
-	t.EventsSkipped = atomic.LoadInt64(&list.eventsSkipped)
-	t.TotalEvents = atomic.LoadInt64(&list.totalEvents)
+	t.CurrentSendCalls = list.currentSendCalls.Load()
+	t.EventsSkipped = list.eventsSkipped.Load()
+	t.TotalEvents = list.totalEvents.Load()
+	t.CurrentQueuedCalls = int64(len(list.queue))
+	t.EventsErrorsTotal = list.eventsErrorsTotal.Load()
 
 	list.RLock()
 	defer list.RUnlock()
@@ -228,7 +264,48 @@ func (list *TargetList) Stats() TargetStats {
 	return t
 }
 
-// NewTargetList - creates TargetList.
-func NewTargetList() *TargetList {
-	return &TargetList{targets: make(map[TargetID]Target)}
+func (list *TargetList) startSendWorkers(workerCount int) {
+	if workerCount == 0 {
+		workerCount = runtime.GOMAXPROCS(0)
+	}
+	wk, err := workers.New(workerCount)
+	if err != nil {
+		panic(err)
+	}
+	for i := 0; i < workerCount; i++ {
+		wk.Take()
+		go func() {
+			defer wk.Give()
+
+			for {
+				select {
+				case av := <-list.queue:
+					list.sendSync(av.ev, av.targetSet)
+				case <-list.ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+	wk.Wait()
+}
+
+var startOnce sync.Once
+
+// Init initialize target send workers.
+func (list *TargetList) Init(workers int) *TargetList {
+	startOnce.Do(func() {
+		go list.startSendWorkers(workers)
+	})
+	return list
+}
+
+// NewTargetList - creates TargetList.
+func NewTargetList(ctx context.Context) *TargetList {
+	list := &TargetList{
+		targets: make(map[TargetID]Target),
+		queue:   make(chan asyncEvent, maxConcurrentAsyncSend),
+		ctx:     ctx,
+	}
+	return list
 }

internal/event/targetlist_test.go

@@ -18,6 +18,7 @@
 package event
 
 import (
+	"context"
 	"crypto/rand"
 	"errors"
 	"reflect"
@@ -85,14 +86,14 @@ func (target ExampleTarget) FlushQueueStore() error {
 }
 
 func TestTargetListAdd(t *testing.T) {
-	targetListCase1 := NewTargetList()
+	targetListCase1 := NewTargetList(context.Background())
 
-	targetListCase2 := NewTargetList()
+	targetListCase2 := NewTargetList(context.Background())
 	if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
 		panic(err)
 	}
 
-	targetListCase3 := NewTargetList()
+	targetListCase3 := NewTargetList(context.Background())
 	if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil {
 		panic(err)
 	}
@@ -140,14 +141,14 @@ func TestTargetListAdd(t *testing.T) {
 }
 
 func TestTargetListExists(t *testing.T) {
-	targetListCase1 := NewTargetList()
+	targetListCase1 := NewTargetList(context.Background())
 
-	targetListCase2 := NewTargetList()
+	targetListCase2 := NewTargetList(context.Background())
 	if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
 		panic(err)
 	}
 
-	targetListCase3 := NewTargetList()
+	targetListCase3 := NewTargetList(context.Background())
 	if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil {
 		panic(err)
 	}
@@ -172,14 +173,14 @@ func TestTargetListExists(t *testing.T) {
 }
 
 func TestTargetListList(t *testing.T) {
-	targetListCase1 := NewTargetList()
+	targetListCase1 := NewTargetList(context.Background())
 
-	targetListCase2 := NewTargetList()
+	targetListCase2 := NewTargetList(context.Background())
 	if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
 		panic(err)
 	}
 
-	targetListCase3 := NewTargetList()
+	targetListCase3 := NewTargetList(context.Background())
 	if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil {
 		panic(err)
 	}
@@ -218,51 +219,8 @@ func TestTargetListList(t *testing.T) {
 	}
 }
 
-func TestTargetListSend(t *testing.T) {
-	targetListCase1 := NewTargetList()
-
-	targetListCase2 := NewTargetList()
-	if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil {
-		panic(err)
-	}
-
-	targetListCase3 := NewTargetList()
-	if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil {
-		panic(err)
-	}
-
-	targetListCase4 := NewTargetList()
-	if err := targetListCase4.Add(&ExampleTarget{TargetID{"4", "testcase"}, true, false}); err != nil {
-		panic(err)
-	}
-
-	testCases := []struct {
-		targetList *TargetList
-		targetID   TargetID
-		expectErr  bool
-	}{
-		{targetListCase1, TargetID{"1", "webhook"}, false},
-		{targetListCase2, TargetID{"1", "non-existent"}, false},
-		{targetListCase3, TargetID{"3", "testcase"}, false},
-		{targetListCase4, TargetID{"4", "testcase"}, true},
-	}
-
-	resCh := make(chan TargetIDResult)
-	for i, testCase := range testCases {
-		testCase.targetList.Send(Event{}, map[TargetID]struct{}{
-			testCase.targetID: {},
-		}, resCh, false)
-		res := <-resCh
-		expectErr := (res.Err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
 func TestNewTargetList(t *testing.T) {
-	if result := NewTargetList(); result == nil {
+	if result := NewTargetList(context.Background()); result == nil {
 		t.Fatalf("test: result: expected: <non-nil>, got: <nil>")
 	}
 }

internal/grid/trace.go

@@ -129,7 +129,11 @@ func (c *muxClient) traceRoundtrip(ctx context.Context, t *tracer, h HandlerID,
 		trace.HTTP.ReqInfo.Path = trace.Path
 	} else if v != nil {
 		// Print exported fields as single request to path.
-		trace.Path = fmt.Sprintf("%s?req=%s", trace.Path, url.QueryEscape(fmt.Sprintf("%+v", v)))
+		obj := fmt.Sprintf("%+v", v)
+		if len(obj) > 1024 {
+			obj = obj[:1024] + "..."
+		}
+		trace.Path = fmt.Sprintf("%s?req=%s", trace.Path, url.QueryEscape(obj))
 		trace.HTTP.ReqInfo.Path = trace.Path
 	}
 	t.Publisher.Publish(trace)

internal/ioutil/ioutil.go

@@ -28,9 +28,39 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/disk"
 )
 
+// Block sizes constant.
+const (
+	BlockSizeSmall       = 32 * humanize.KiByte // Default r/w block size for smaller objects.
+	BlockSizeLarge       = 2 * humanize.MiByte  // Default r/w block size for larger objects.
+	BlockSizeReallyLarge = 4 * humanize.MiByte  // Default write block size for objects per shard >= 64MiB
+)
+
+// aligned sync.Pool's
+var (
+	ODirectPoolXLarge = sync.Pool{
+		New: func() interface{} {
+			b := disk.AlignedBlock(BlockSizeReallyLarge)
+			return &b
+		},
+	}
+	ODirectPoolLarge = sync.Pool{
+		New: func() interface{} {
+			b := disk.AlignedBlock(BlockSizeLarge)
+			return &b
+		},
+	}
+	ODirectPoolSmall = sync.Pool{
+		New: func() interface{} {
+			b := disk.AlignedBlock(BlockSizeSmall)
+			return &b
+		},
+	}
+)
+
 // WriteOnCloser implements io.WriteCloser and always
 // executes at least one write operation if it is closed.
 //

internal/ioutil/odirect_reader.go

@@ -1,136 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package ioutil
-
-import (
-	"errors"
-	"io"
-	"os"
-	"sync"
-	"syscall"
-
-	"github.com/dustin/go-humanize"
-	"github.com/minio/minio/internal/disk"
-)
-
-// ODirectReader - to support O_DIRECT reads for erasure backends.
-type ODirectReader struct {
-	File      *os.File
-	SmallFile bool
-	err       error
-	buf       []byte
-	bufp      *[]byte
-	seenRead  bool
-}
-
-// Block sizes constant.
-const (
-	BlockSizeSmall       = 32 * humanize.KiByte // Default r/w block size for smaller objects.
-	BlockSizeLarge       = 2 * humanize.MiByte  // Default r/w block size for larger objects.
-	BlockSizeReallyLarge = 4 * humanize.MiByte  // Default write block size for objects per shard >= 64MiB
-)
-
-// O_DIRECT aligned sync.Pool's
-var (
-	ODirectPoolXLarge = sync.Pool{
-		New: func() interface{} {
-			b := disk.AlignedBlock(BlockSizeReallyLarge)
-			return &b
-		},
-	}
-	ODirectPoolLarge = sync.Pool{
-		New: func() interface{} {
-			b := disk.AlignedBlock(BlockSizeLarge)
-			return &b
-		},
-	}
-	ODirectPoolSmall = sync.Pool{
-		New: func() interface{} {
-			b := disk.AlignedBlock(BlockSizeSmall)
-			return &b
-		},
-	}
-)
-
-// Invalid argument, unsupported flags such as O_DIRECT
-func isSysErrInvalidArg(err error) bool {
-	return errors.Is(err, syscall.EINVAL)
-}
-
-// Read - Implements Reader interface.
-func (o *ODirectReader) Read(buf []byte) (n int, err error) {
-	if o.err != nil && (len(o.buf) == 0 || !o.seenRead) {
-		return 0, o.err
-	}
-	if o.buf == nil {
-		if o.SmallFile {
-			o.bufp = ODirectPoolSmall.Get().(*[]byte)
-		} else {
-			o.bufp = ODirectPoolLarge.Get().(*[]byte)
-		}
-	}
-	if !o.seenRead {
-		o.buf = *o.bufp
-		n, err = o.File.Read(o.buf)
-		if err != nil && err != io.EOF {
-			if isSysErrInvalidArg(err) {
-				if err = disk.DisableDirectIO(o.File); err != nil {
-					o.err = err
-					return n, err
-				}
-				n, err = o.File.Read(o.buf)
-			}
-			if err != nil && err != io.EOF {
-				o.err = err
-				return n, err
-			}
-		}
-		if n == 0 {
-			// err is likely io.EOF
-			o.err = err
-			return n, err
-		}
-		o.err = err
-		o.buf = o.buf[:n]
-		o.seenRead = true
-	}
-	if len(buf) >= len(o.buf) {
-		n = copy(buf, o.buf)
-		o.seenRead = false
-		return n, o.err
-	}
-	n = copy(buf, o.buf)
-	o.buf = o.buf[n:]
-	// There is more left in buffer, do not return any EOF yet.
-	return n, nil
-}
-
-// Close - Release the buffer and close the file.
-func (o *ODirectReader) Close() error {
-	if o.bufp != nil {
-		if o.SmallFile {
-			ODirectPoolSmall.Put(o.bufp)
-		} else {
-			ODirectPoolLarge.Put(o.bufp)
-		}
-		o.bufp = nil
-		o.buf = nil
-	}
-	o.err = errors.New("internal error: ODirectReader Read after Close")
-	return o.File.Close()
-}

internal/ioutil/read_file.go

@@ -62,48 +62,18 @@ func ReadFileWithFileInfo(name string) ([]byte, fs.FileInfo, error) {
 //
 // passes NOATIME flag for reads on Unix systems to avoid atime updates.
 func ReadFile(name string) ([]byte, error) {
-	if !disk.ODirectPlatform {
-		// Don't wrap with un-needed buffer.
-		// Don't use os.ReadFile, since it doesn't pass NO_ATIME when present.
-		f, err := OsOpenFile(name, readMode, 0o666)
-		if err != nil {
-			return nil, err
-		}
-		defer f.Close()
-		st, err := f.Stat()
-		if err != nil {
-			return io.ReadAll(f)
-		}
-		dst := make([]byte, st.Size())
-		_, err = io.ReadFull(f, dst)
-		return dst, err
-	}
-
-	f, err := OpenFileDirectIO(name, readMode, 0o666)
+	// Don't wrap with un-needed buffer.
+	// Don't use os.ReadFile, since it doesn't pass NO_ATIME when present.
+	f, err := OsOpenFile(name, readMode, 0o666)
 	if err != nil {
-		// fallback if there is an error to read
-		// 'name' with O_DIRECT
-		f, err = OsOpenFile(name, readMode, 0o666)
-		if err != nil {
-			return nil, err
-		}
+		return nil, err
 	}
-
-	r := &ODirectReader{
-		File:      f,
-		SmallFile: true,
-	}
-	defer r.Close()
-
+	defer f.Close()
 	st, err := f.Stat()
 	if err != nil {
-		return io.ReadAll(r)
+		return io.ReadAll(f)
 	}
-
-	// Select bigger blocks when reading would do more than 2 reads.
-	r.SmallFile = st.Size() <= BlockSizeSmall*2
-
 	dst := make([]byte, st.Size())
-	_, err = io.ReadFull(r, dst)
+	_, err = io.ReadFull(f, dst)
 	return dst, err
 }