Add decommission compression+encryption enabled tests (#15322)

update compression environment variables to follow
the expected sub-system style, however support fallback
mode.

.github/workflows/go-healing.yml

@@ -48,3 +48,4 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify-healing
           make verify-healing-inconsistent-versions
+          make verify-healing-with-root-disks

.github/workflows/markdown-lint.yaml

@@ -27,4 +27,4 @@ jobs:
         npm install -g markdownlint-cli
         markdownlint --fix '**/*.md' \
           --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
-          --disable MD013 MD040
+          --disable MD013 MD040 MD051

Makefile

@@ -44,6 +44,9 @@ test: verifiers build ## builds minio, runs linters, tests
 test-decom: install
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
+	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
 test-upgrade: build
 	@echo "Running minio upgrade tests"
@@ -87,6 +90,11 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
+verify-healing-with-root-disks:
+	@echo "Verify healing with root disks"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
+
 verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

README.fips.md

@@ -2,6 +2,6 @@
 
 MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring).
 
-MinIO FIPS executables are available at http://dl.min.io - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
+MinIO FIPS executables are available at <http://dl.min.io> - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
 
 We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes.

buildscripts/verify-healing-with-root-disks.sh

@@ -0,0 +1,97 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$(mktemp -d)"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+
+function start_minio() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    unset MINIO_CI_CD
+    unset CI
+
+    args=()
+    for i in $(seq 1 4); do
+            args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ")
+    done
+
+    for i in $(seq 1 4); do
+            "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
+    done
+
+    # Wait until all nodes return 403
+    for i in $(seq 1 4); do
+	    while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do
+		    echo -n ".";
+                    sleep 1;
+            done
+    done
+
+ }
+
+# Prepare fake disks with losetup
+function prepare_block_devices() {
+        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+        for i in 1 2 3 4; do
+                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
+                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
+                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
+                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
+                mkdir -p ${WORK_DIR}/mnt/disk$i/
+                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+        done
+}
+
+# Start a distributed MinIO setup, unmount one disk and check if it is formatted
+function main() {
+    start_port=$(shuf -i 10000-65000 -n 1)
+    start_minio ${start_port}
+
+    # Unmount the disk, after the unmount the device id
+    # /tmp/xxx/mnt/disk4 will be the same as '/' and it
+    # will be detected as root disk
+    while [ "$u" != "0" ]; do
+	    sudo umount ${WORK_DIR}/mnt/disk4/
+	    u=$?
+	    sleep 1
+    done
+
+    # Wait until MinIO self heal kicks in
+    sleep 60
+
+    if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
+	    echo "A root disk is formatted unexpectedely"
+            cat "${WORK_DIR}/server4.log"
+            exit -1
+    fi
+}
+
+function cleanup() {
+    pkill minio
+    sudo umount ${WORK_DIR}/mnt/disk{1..3}/
+    sudo rm /dev/minio-loopdisk*
+    rm -rf "$WORK_DIR"
+}
+
+( prepare_block_devices )
+( main "$@" )
+rv=$?
+
+cleanup
+exit "$rv"
+

cmd/admin-handlers.go

@@ -1149,17 +1149,17 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
-func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	a.ObjectSpeedtestHandler(w, r)
+// SpeedtestHandler - Deprecated. See ObjectSpeedTestHandler
+func (a adminAPIHandlers) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
+	a.ObjectSpeedTestHandler(w, r)
 }
 
-// ObjectSpeedtestHandler - reports maximum speed of a cluster by performing PUT and
+// ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and
 // GET operations on the server, supports auto tuning by default by automatically
 // increasing concurrency and stopping when we have reached the limits on the
 // system.
-func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ObjectSpeedtestHandler")
+func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ObjectSpeedTestHandler")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -1176,8 +1176,9 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
+	storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass))
+	customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
 	autotune := r.Form.Get("autotune") == "true"
-	storageClass := r.Form.Get("storage-class")
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1212,15 +1213,23 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		autotune = false
 	}
 
-	bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
-		return
+	if customBucket == "" {
+		customBucket = globalObjectPerfBucket
+
+		bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, customBucket)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+
+		if !bucketExists {
+			defer deleteObjectPerfBucket(objectAPI)
+		}
 	}
 
-	if !bucketExists {
-		defer deleteObjectPerfBucket(objectAPI)
-	}
+	defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
+		DeletePrefix: true,
+	})
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1232,7 +1241,14 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
-	ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
+	ch := objectSpeedTest(ctx, speedTestOpts{
+		objectSize:       size,
+		concurrencyStart: concurrent,
+		duration:         duration,
+		autotune:         autotune,
+		storageClass:     storageClass,
+		bucketName:       customBucket,
+	})
 	for {
 		select {
 		case <-ctx.Done():
@@ -1255,9 +1271,8 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	}
 }
 
-func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) {
-	err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
-	if err != nil {
+func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
+	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
@@ -2203,9 +2218,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				autotune = false
 			}
 
-			bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
+			bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, globalObjectPerfBucket)
 			if err != nil {
-				healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error()
+				healthInfo.Perf.Error = "Unable to create bucket: " + err.Error()
 				partialWrite(healthInfo)
 				return
 			}
@@ -2215,7 +2230,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 
 			opts := speedTestOpts{
-				throughputSize:   size,
+				objectSize:       size,
 				concurrencyStart: concurrent,
 				duration:         10 * time.Second,
 				autotune:         autotune,

cmd/admin-router.go

@@ -244,8 +244,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
 		}
 
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
 

cmd/background-newdisks-heal-ops.go

@@ -94,7 +94,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 		return nil, err
 	}
 	b, err := disk.ReadAll(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename))
+		pathJoin(bucketMetaPrefix, healingTrackerFilename))
 	if err != nil {
 		return nil, err
 	}
@@ -155,15 +155,19 @@ func (h *healingTracker) save(ctx context.Context) error {
 	}
 	globalBackgroundHealState.updateHealStatus(h)
 	return h.disk.WriteAll(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+		pathJoin(bucketMetaPrefix, healingTrackerFilename),
 		htrackerBytes)
 }
 
 // delete the tracker on disk.
 func (h *healingTracker) delete(ctx context.Context) error {
 	return h.disk.Delete(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
-		false)
+		pathJoin(bucketMetaPrefix, healingTrackerFilename),
+		DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		},
+	)
 }
 
 func (h *healingTracker) isHealed(bucket string) bool {
@@ -361,12 +365,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	// Start or resume healing of this erasure set
-	err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker)
-	if err != nil {
+	if err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker); err != nil {
 		return err
 	}
 
-	logger.Info("Healing disk '%s' is complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	if tracker.ItemsFailed > 0 {
+		logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	} else {
+		logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	}
 
 	if serverDebugLog {
 		tracker.printTo(os.Stdout)
@@ -374,6 +381,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	logger.LogIf(ctx, tracker.delete(ctx))
+
 	return nil
 }
 
@@ -390,12 +398,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 		case <-ctx.Done():
 			return
 		case <-diskCheckTimer.C:
-
 			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
 			if len(healDisks) == 0 {
 				// Reset for next interval.
 				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
-				break
+				continue
 			}
 
 			// Reformat disks immediately
@@ -404,7 +411,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 				logger.LogIf(ctx, err)
 				// Reset for next interval.
 				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
-				break
+				continue
 			}
 
 			for _, disk := range healDisks {

cmd/bitrot-streaming.go

@@ -152,10 +152,10 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		if len(b.data) == 0 && b.tillOffset != streamOffset {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
-			if err != nil {
+			if err != nil && err != errDiskNotFound {
 				logger.LogIf(GlobalContext,
-					fmt.Errorf("Error(%w) reading erasure shards at (%s: %s/%s), will attempt to reconstruct if we have quorum",
-						err, b.disk, b.volume, b.filePath))
+					fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
+						b.disk, b.volume, b.filePath, err))
 			}
 		} else {
 			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)

cmd/bucket-handlers.go

@@ -649,6 +649,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			dv := DeletedObjectReplicationInfo{
 				DeletedObject: dobj,
 				Bucket:        bucket,
+				EventType:     ReplicateIncomingDelete,
 			}
 			scheduleReplicationDelete(ctx, dv, objectAPI)
 		}
@@ -864,7 +865,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1040,7 +1041,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if crypto.Requested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
 				return
@@ -1115,9 +1116,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if successRedirect != "" {
-		// Replace raw query params..
-		redirectURL.RawQuery = getRedirectPostRawQuery(objInfo)
+	if redirectURL != nil { // success_action_redirect is valid and set.
+		v := redirectURL.Query()
+		v.Add("bucket", objInfo.Bucket)
+		v.Add("key", objInfo.Name)
+		v.Add("etag", "\""+objInfo.ETag+"\"")
+		redirectURL.RawQuery = v.Encode()
 		writeRedirectSeeOther(w, redirectURL.String())
 		return
 	}

cmd/bucket-replication.go

@@ -347,7 +347,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
 // deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
 // on target.
-func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, trigger string) {
+func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
 	var replicationStatus replication.StatusType
 	bucket := dobj.Bucket
 	versionID := dobj.DeleteMarkerVersionID
@@ -358,7 +358,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	defer func() {
 		replStatus := string(replicationStatus)
 		auditLogInternal(context.Background(), bucket, dobj.ObjectName, AuditLogOptions{
-			Trigger:   trigger,
+			Event:     dobj.EventType,
 			APIName:   ReplicateDeleteAPI,
 			VersionID: versionID,
 			Status:    replStatus,
@@ -857,7 +857,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
-func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer, trigger string) {
+func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) {
 	var replicationStatus replication.StatusType
 	defer func() {
 		if replicationStatus.Empty() {
@@ -868,7 +868,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			replicationStatus = ri.ReplicationStatus
 		}
 		auditLogInternal(ctx, ri.Bucket, ri.Name, AuditLogOptions{
-			Trigger:   trigger,
+			Event:     ri.EventType,
 			APIName:   ReplicateObjectAPI,
 			VersionID: ri.VersionID,
 			Status:    replicationStatus.String(),
@@ -990,6 +990,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	// the target site is down. Leave it to scanner to catch up instead.
 	if rinfos.ReplicationStatus() != replication.Completed && ri.RetryCount < 1 {
 		ri.OpType = replication.HealReplicationType
+		ri.EventType = ReplicateMRF
 		ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
 		ri.RetryCount++
 		globalReplicationPool.queueReplicaFailedTask(ri)
@@ -1291,6 +1292,7 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 type DeletedObjectReplicationInfo struct {
 	DeletedObject
 	Bucket    string
+	EventType string
 	OpType    replication.Type
 	ResetID   string
 	TargetArn string
@@ -1313,12 +1315,15 @@ const (
 
 	// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
 	ReplicateMRF = "replicate:mrf"
-	// ReplicateIncoming - audit trail indicating replication started [could be from incoming/existing/heal activity]
+	// ReplicateIncoming - audit trail of inline replication
 	ReplicateIncoming = "replicate:incoming"
+	// ReplicateIncomingDelete - audit trail of inline replication of deletes.
+	ReplicateIncomingDelete = "replicate:incoming:delete"
+
 	// ReplicateHeal - audit trail for healing of failed/pending replications
 	ReplicateHeal = "replicate:heal"
-	// ReplicateDelete - audit trail for delete replication
-	ReplicateDelete = "replicate:delete"
+	// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
+	ReplicateHealDelete = "replicate:heal:delete"
 )
 
 var (
@@ -1364,13 +1369,14 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool.ResizeWorkers(opts.Workers)
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
 	go pool.AddExistingObjectReplicateWorker()
-	go pool.periodicResyncMetaSave(ctx, o)
+	go pool.updateResyncStatus(ctx, o)
 	return pool
 }
 
 // AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued
 // to the other workers
 func (p *ReplicationPool) AddMRFWorker() {
+	defer p.mrfWorkerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1379,7 +1385,7 @@ func (p *ReplicationPool) AddMRFWorker() {
 			if !ok {
 				return
 			}
-			replicateObject(p.ctx, oi, p.objLayer, ReplicateMRF)
+			replicateObject(p.ctx, oi, p.objLayer)
 		case <-p.mrfWorkerKillCh:
 			return
 		}
@@ -1397,12 +1403,12 @@ func (p *ReplicationPool) AddWorker() {
 			if !ok {
 				return
 			}
-			replicateObject(p.ctx, oi, p.objLayer, ReplicateIncoming)
+			replicateObject(p.ctx, oi, p.objLayer)
 		case doi, ok := <-p.replicaDeleteCh:
 			if !ok {
 				return
 			}
-			replicateDelete(p.ctx, doi, p.objLayer, ReplicateDelete)
+			replicateDelete(p.ctx, doi, p.objLayer)
 		case <-p.workerKillCh:
 			return
 		}
@@ -1419,12 +1425,12 @@ func (p *ReplicationPool) AddExistingObjectReplicateWorker() {
 			if !ok {
 				return
 			}
-			replicateObject(p.ctx, oi, p.objLayer, ReplicateExisting)
+			replicateObject(p.ctx, oi, p.objLayer)
 		case doi, ok := <-p.existingReplicaDeleteCh:
 			if !ok {
 				return
 			}
-			replicateDelete(p.ctx, doi, p.objLayer, ReplicateExistingDelete)
+			replicateDelete(p.ctx, doi, p.objLayer)
 		}
 	}
 }
@@ -1482,7 +1488,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication failed workers could not keep up with healing failures - consider increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
 	}
 }
 
@@ -1508,7 +1514,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1545,7 +1551,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1722,10 +1728,11 @@ func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs
 }
 
 func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, dsc ReplicateDecision, opType replication.Type) {
+	ri := ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc, EventType: ReplicateIncoming}
 	if dsc.Synchronous() {
-		replicateObject(ctx, ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc}, o, ReplicateIncoming)
+		replicateObject(ctx, ri, o)
 	} else {
-		globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc})
+		globalReplicationPool.queueReplicaTask(ri)
 	}
 	if sz, err := objInfo.GetActualSize(); err == nil {
 		for arn := range dsc.targetsMap {
@@ -1956,10 +1963,10 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	return calculateBucketReplicationStats(bucket, u, bucketStats)
 }
 
-const resyncTimeInterval = time.Minute * 10
+const resyncTimeInterval = time.Minute * 1
 
-// periodicResyncMetaSave saves in-memory resync meta stats to disk in periodic intervals
-func (p *ReplicationPool) periodicResyncMetaSave(ctx context.Context, objectAPI ObjectLayer) {
+// updateResyncStatus persists in-memory resync metadata stats to disk at periodic intervals
+func (p *ReplicationPool) updateResyncStatus(ctx context.Context, objectAPI ObjectLayer) {
 	resyncTimer := time.NewTimer(resyncTimeInterval)
 	defer resyncTimer.Stop()
 
@@ -2085,13 +2092,15 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI
 					DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 					DeleteMarker:          roi.DeleteMarker,
 				},
-				Bucket: roi.Bucket,
-				OpType: replication.ExistingObjectReplicationType,
+				Bucket:    roi.Bucket,
+				OpType:    replication.ExistingObjectReplicationType,
+				EventType: ReplicateExistingDelete,
 			}
-			replicateDelete(ctx, doi, objectAPI, ReplicateDelete)
+			replicateDelete(ctx, doi, objectAPI)
 		} else {
 			roi.OpType = replication.ExistingObjectReplicationType
-			replicateObject(ctx, roi, objectAPI, ReplicateExisting)
+			roi.EventType = ReplicateExisting
+			replicateObject(ctx, roi, objectAPI)
 		}
 		_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, miniogo.StatObjectOptions{
 			VersionID: roi.VersionID,
@@ -2205,18 +2214,21 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo,
 	for index := range buckets {
 		meta, err := loadBucketResyncMetadata(ctx, buckets[index].Name, objAPI)
 		if err != nil {
-			if errors.Is(err, errVolumeNotFound) {
-				meta = newBucketResyncStatus(buckets[index].Name)
-			} else {
+			if !errors.Is(err, errVolumeNotFound) {
 				logger.LogIf(ctx, err)
-				continue
 			}
+			continue
 		}
+		p.resyncState.Lock()
 		p.resyncState.statusMap[buckets[index].Name] = meta
+		p.resyncState.Unlock()
 	}
 	for index := range buckets {
 		bucket := buckets[index].Name
+		p.resyncState.RLock()
 		m, ok := p.resyncState.statusMap[bucket]
+		p.resyncState.RUnlock()
+
 		if ok {
 			for arn, st := range m.TargetsMap {
 				if st.ResyncStatus == ResyncFailed || st.ResyncStatus == ResyncStarted {

cmd/common-main.go

@@ -430,7 +430,6 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		if err != nil {
 			logger.FatalIf(err, "Unable to get free port for console on the host")
 		}
-		globalMinioConsolePortAuto = true
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
 

cmd/data-scanner.go

@@ -48,12 +48,13 @@ import (
 )
 
 const (
-	dataScannerSleepPerFolder     = time.Millisecond                 // Time to wait between folders.
-	dataUsageUpdateDirCycles      = 16                               // Visit all folders every n cycles.
-	dataScannerCompactLeastObject = 500                              // Compact when there is less than this many objects in a branch.
-	dataScannerCompactAtChildren  = 10000                            // Compact when there are this many children in a branch.
-	dataScannerCompactAtFolders   = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
-	dataScannerStartDelay         = 1 * time.Minute                  // Time to wait on startup and between cycles.
+	dataScannerSleepPerFolder        = time.Millisecond                 // Time to wait between folders.
+	dataUsageUpdateDirCycles         = 16                               // Visit all folders every n cycles.
+	dataScannerCompactLeastObject    = 500                              // Compact when there is less than this many objects in a branch.
+	dataScannerCompactAtChildren     = 10000                            // Compact when there are this many children in a branch.
+	dataScannerCompactAtFolders      = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
+	dataScannerForceCompactAtFolders = 1_000_000                        // Compact when this many subfolders in a single folder (even top level).
+	dataScannerStartDelay            = 1 * time.Minute                  // Time to wait on startup and between cycles.
 
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
@@ -566,9 +567,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		}
 
 		// If we have many subfolders, compact ourself.
-		if !into.Compacted &&
-			f.newCache.Info.Name != folder.name &&
-			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders {
+		shouldCompact := f.newCache.Info.Name != folder.name &&
+			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
+			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
+
+		if !into.Compacted && shouldCompact {
 			into.Compacted = true
 			newFolders = append(newFolders, existingFolders...)
 			existingFolders = nil
@@ -734,11 +737,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 				},
 				// Some disks have data for this.
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-					if f.dataUsageScannerDebug {
-						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
-					}
-
+				partial: func(entries metaCacheEntries, errs []error) {
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						// check if we can get one entry atleast
@@ -1279,12 +1278,14 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 
 	switch oi.ReplicationStatus {
 	case replication.Pending, replication.Failed:
+		roi.EventType = ReplicateHeal
 		globalReplicationPool.queueReplicaTask(roi)
 		return
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
 	if roi.ExistingObjResync.mustResync() {
+		roi.EventType = ReplicateExisting
 		globalReplicationPool.queueReplicaTask(roi)
 	}
 }
@@ -1310,11 +1311,13 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},
-			Bucket: roi.Bucket,
-			OpType: replication.HealReplicationType,
+			Bucket:    roi.Bucket,
+			OpType:    replication.HealReplicationType,
+			EventType: ReplicateHealDelete,
 		}
 		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType
+			doi.EventType = ReplicateExistingDelete
 			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
 			return
 		}
@@ -1477,9 +1480,9 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 	var apiName string
-	switch trigger {
+	switch event {
 	case ILMExpiry:
 		apiName = "ILMExpiry"
 	case ILMFreeVersionDelete:
@@ -1488,7 +1491,7 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
 		apiName = "ILMTransition"
 	}
 	auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
-		Trigger:   trigger,
+		Event:     event,
 		APIName:   apiName,
 		VersionID: oi.VersionID,
 	})

cmd/erasure-encode_test.go

@@ -196,7 +196,10 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 			if disk == OfflineDisk {
 				continue
 			}
-			disk.Delete(context.Background(), "testbucket", "object", false)
+			disk.Delete(context.Background(), "testbucket", "object", DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

cmd/erasure-healing-common_test.go

@@ -218,7 +218,10 @@ func TestListOnlineDisks(t *testing.T) {
 					// and check if that disk
 					// appears in outDatedDisks.
 					tamperedIndex = index
-					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+						Recursive: false,
+						Force:     false,
+					})
 					if dErr != nil {
 						t.Fatalf("Failed to delete %s - %v", filepath.Join(object, "part.1"), dErr)
 					}
@@ -395,7 +398,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					// and check if that disk
 					// appears in outDatedDisks.
 					tamperedIndex = index
-					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+						Recursive: false,
+						Force:     false,
+					})
 					if dErr != nil {
 						t.Fatalf("Failed to delete %s - %v", pathJoin(object, xlStorageFormatFile), dErr)
 					}

cmd/erasure-healing.go

@@ -501,6 +501,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			partSize := latestMeta.Parts[partIndex].Size
 			partActualSize := latestMeta.Parts[partIndex].ActualSize
 			partNumber := latestMeta.Parts[partIndex].Number
+			partIdx := latestMeta.Parts[partIndex].Index
 			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
@@ -550,7 +551,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 
 				partsMetadata[i].DataDir = dstDataDir
-				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize)
+				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partIdx)
 				partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 					PartNumber: partNumber,
 					Algorithm:  checksumAlgo,
@@ -655,7 +656,10 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				wg.Add(1)
 				go func(index int, disk StorageAPI) {
 					defer wg.Done()
-					_ = disk.Delete(ctx, bucket, object, false)
+					_ = disk.Delete(ctx, bucket, object, DeleteOptions{
+						Recursive: false,
+						Force:     false,
+					})
 				}(index, disk)
 			}
 			wg.Wait()

cmd/erasure-healing_test.go

@@ -610,7 +610,10 @@ func TestHealCorrectQuorum(t *testing.T) {
 		}
 
 		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
-			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 		}
 
 		// Try healing now, it should heal the content properly.
@@ -634,7 +637,10 @@ func TestHealCorrectQuorum(t *testing.T) {
 		}
 
 		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
-			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), false)
+			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 		}
 
 		// Try healing now, it should heal the content properly.
@@ -714,7 +720,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	er := set.sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -734,7 +743,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
@@ -761,7 +773,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
@@ -792,7 +807,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	// Test 4: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -884,7 +902,10 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -936,7 +957,10 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	// Test 3: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= nfi2.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -1033,7 +1057,10 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	}
 
 	// Test 1, remove part.1
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -1078,7 +1105,10 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	err = secondDisk.Delete(context.Background(), bucket, object, true)
+	err = secondDisk.Delete(context.Background(), bucket, object, DeleteOptions{
+		Recursive: true,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -1166,7 +1196,10 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	// Delete the whole object folder
-	err = firstDisk.Delete(context.Background(), bucket, object, true)
+	err = firstDisk.Delete(context.Background(), bucket, object, DeleteOptions{
+		Recursive: true,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}

cmd/erasure-metadata.go

@@ -237,12 +237,13 @@ func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 }
 
 // AddObjectPart - add a new object part in order.
-func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize int64, actualSize int64) {
+func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, idx []byte) {
 	partInfo := ObjectPartInfo{
 		Number:     partNumber,
 		ETag:       partETag,
 		Size:       partSize,
 		ActualSize: actualSize,
+		Index:      idx,
 	}
 
 	// Update part info if it already exists.

cmd/erasure-metadata_test.go

@@ -23,7 +23,7 @@ import (
 	"testing"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 const ActualSize = 1000
@@ -58,7 +58,7 @@ func TestAddObjectPart(t *testing.T) {
 	for _, testCase := range testCases {
 		if testCase.expectedIndex > -1 {
 			partNumString := strconv.Itoa(testCase.partNum)
-			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, nil)
 		}
 
 		if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
@@ -91,7 +91,7 @@ func TestObjectPartIndex(t *testing.T) {
 	// Add some parts for testing.
 	for _, testCase := range testCases {
 		partNumString := strconv.Itoa(testCase.partNum)
-		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, nil)
 	}
 
 	// Add failure test case.
@@ -121,7 +121,7 @@ func TestObjectToPartOffset(t *testing.T) {
 	// Total size of all parts is 5,242,899 bytes.
 	for _, partNum := range []int{1, 2, 4, 5, 7} {
 		partNumString := strconv.Itoa(partNum)
-		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize)
+		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize, nil)
 	}
 
 	testCases := []struct {
@@ -160,7 +160,7 @@ func TestObjectToPartOffset(t *testing.T) {
 func TestFindFileInfoInQuorum(t *testing.T) {
 	getNFInfo := func(n int, quorum int, t int64, dataDir string) []FileInfo {
 		fi := newFileInfo("test", 8, 8)
-		fi.AddObjectPart(1, "etag", 100, 100)
+		fi.AddObjectPart(1, "etag", 100, 100, nil)
 		fi.ModTime = time.Unix(t, 0)
 		fi.DataDir = dataDir
 		fis := make([]FileInfo, n)

cmd/erasure-multipart.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -48,7 +49,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
 func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) (err error) {
 	defer func() {
-		if err == errFileNotFound {
+		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
 			err = errUploadIDNotFound
 		}
 	}()
@@ -91,7 +92,10 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 			return nil
 		}, index)
 	}
@@ -138,7 +142,10 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 		wg.Add(1)
 		go func(disk StorageAPI) {
 			defer wg.Done()
-			disk.Delete(ctx, bucket, prefix, true)
+			disk.Delete(ctx, bucket, prefix, DeleteOptions{
+				Recursive: true,
+				Force:     false,
+			})
 		}(disk)
 	}
 	wg.Wait()
@@ -202,10 +209,10 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 	for _, disk = range er.getLoadBalancedDisks(true) {
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
-			if err == errDiskNotFound {
+			if errors.Is(err, errDiskNotFound) {
 				continue
 			}
-			if err == errFileNotFound {
+			if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
 				return result, nil
 			}
 			logger.LogIf(ctx, err)
@@ -230,7 +237,11 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		}
 		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "", false)
 		if err != nil {
-			return result, toObjectErr(err, bucket, object)
+			if !IsErrIgnored(err, errFileNotFound, errDiskNotFound) {
+				logger.LogIf(ctx, err)
+			}
+			// Ignore this invalid upload-id since we are listing here
+			continue
 		}
 		populatedUploadIds.Add(uploadID)
 		uploads = append(uploads, MultipartInfo{
@@ -647,9 +658,16 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	fi.ModTime = UTCNow()
 
 	md5hex := r.MD5CurrentHexString()
+	if opts.PreserveETag != "" {
+		md5hex = opts.PreserveETag
+	}
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
 
 	// Add the current part.
-	fi.AddObjectPart(partID, md5hex, n, data.ActualSize())
+	fi.AddObjectPart(partID, md5hex, n, data.ActualSize(), index)
 
 	for i, disk := range onlineDisks {
 		if disk == OfflineDisk {
@@ -941,6 +959,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			Number:     part.PartNumber,
 			Size:       currentFI.Parts[partIdx].Size,
 			ActualSize: currentFI.Parts[partIdx].ActualSize,
+			Index:      currentFI.Parts[partIdx].Index,
 		}
 	}
 

cmd/erasure-object.go

@@ -815,6 +815,10 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	if n < data.Size() {
 		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
 	}
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
 
 	for i, w := range writers {
 		if w == nil {
@@ -823,7 +827,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -972,14 +976,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	switch size := data.Size(); {
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
-	case size == -1:
-		if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
-			buffer = make([]byte, data.ActualSize()+256, data.ActualSize()*2+512)
-		} else {
-			buffer = er.bp.Get()
-			defer er.bp.Put(buffer)
-		}
-	case size >= fi.Erasure.BlockSize:
+	case size >= fi.Erasure.BlockSize || size == -1:
 		buffer = er.bp.Get()
 		defer er.bp.Put(buffer)
 	case size < fi.Erasure.BlockSize:
@@ -1071,6 +1068,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
+	var compIndex []byte
+	if opts.IndexCB != nil {
+		compIndex = opts.IndexCB()
+	}
 	if !opts.NoLock {
 		lk := er.NewNSLock(bucket, object)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -1091,15 +1092,19 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), compIndex)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
 			Hash:       bitrotWriterSum(w),
 		})
 	}
+
 	if userDefined["etag"] == "" {
 		userDefined["etag"] = r.MD5CurrentHexString()
+		if opts.PreserveETag != "" {
+			userDefined["etag"] = opts.PreserveETag
+		}
 	}
 
 	// Guess content-type from the extension if possible.
@@ -1367,8 +1372,14 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 			// Deletes
 			// - The prefix and its children
 			// - The prefix__XLDIR__
-			defer disks[index].Delete(ctx, bucket, dirPrefix, true)
-			return disks[index].Delete(ctx, bucket, prefix, true)
+			defer disks[index].Delete(ctx, bucket, dirPrefix, DeleteOptions{
+				Recursive: true,
+				Force:     true,
+			})
+			return disks[index].Delete(ctx, bucket, prefix, DeleteOptions{
+				Recursive: true,
+				Force:     true,
+			})
 		}, index)
 	}
 	for _, err := range g.Wait() {

cmd/erasure-object_test.go

@@ -528,7 +528,10 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		files, _ := disk.ListDir(ctx, bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
-				disk.Delete(ctx, bucket, pathJoin(object, file), true)
+				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
+					Recursive: true,
+					Force:     false,
+				})
 			}
 		}
 	}
@@ -629,7 +632,10 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 		files, _ := disk.ListDir(ctx, bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
-				disk.Delete(ctx, bucket, pathJoin(object, file), true)
+				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
+					Recursive: true,
+					Force:     false,
+				})
 			}
 		}
 	}

cmd/erasure-server-pool-decom.go

@@ -22,6 +22,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"sort"
 	"strconv"
@@ -30,6 +31,8 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
@@ -523,14 +526,26 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 			}
 			if globalEndpoints[idx].Endpoints[0].IsLocal {
 				go func(pool PoolStatus) {
-					switch err := z.Decommission(ctx, pool.ID); err {
-					case nil:
-						// we already started decommission
-					case errDecommissionAlreadyRunning:
-						// A previous decommission running found restart it.
-						z.doDecommissionInRoutine(ctx, idx)
-					default:
-						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+					r := rand.New(rand.NewSource(time.Now().UnixNano()))
+					for {
+						if err := z.Decommission(ctx, pool.ID); err != nil {
+							switch err {
+							// we already started decommission
+							case errDecommissionAlreadyRunning:
+								// A previous decommission running found restart it.
+								z.doDecommissionInRoutine(ctx, idx)
+								return
+							default:
+								if configRetriableErrors(err) {
+									logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w: retrying..", pool, err))
+									time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
+									continue
+								}
+								logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+								return
+							}
+						}
+						break
 					}
 				}(pool)
 			}
@@ -566,6 +581,11 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		auditLogDecom(ctx, "DecomCopyData", objInfo.Bucket, objInfo.Name, objInfo.VersionID, err)
 	}()
 
+	actualSize, err := objInfo.GetActualSize()
+	if err != nil {
+		return err
+	}
+
 	if objInfo.isMultipart() {
 		uploadID, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
@@ -573,21 +593,26 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			UserDefined: objInfo.UserDefined,
 		})
 		if err != nil {
-			return err
+			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
 		}
 		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, uploadID, ObjectOptions{})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
-			hr, err := hash.NewReader(gr, part.Size, "", "", part.Size)
+			hr, err := hash.NewReader(gr, part.Size, "", "", part.ActualSize)
 			if err != nil {
-				return err
+				return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 			}
 			pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, uploadID,
 				part.Number,
 				NewPutObjReader(hr),
-				ObjectOptions{})
+				ObjectOptions{
+					PreserveETag: part.ETag, // Preserve original ETag to ensure same metadata.
+					IndexCB: func() []byte {
+						return part.Index // Preserve part Index to ensure decompression works.
+					},
+				})
 			if err != nil {
-				return err
+				return fmt.Errorf("decommissionObject: PutObjectPart() %w", err)
 			}
 			parts[i] = CompletePart{
 				ETag:       pi.ETag,
@@ -597,21 +622,32 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, uploadID, parts, ObjectOptions{
 			MTime: objInfo.ModTime,
 		})
+		if err != nil {
+			err = fmt.Errorf("decommissionObject: CompleteMultipartUpload() %w", err)
+		}
 		return err
 	}
-	hr, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size)
+
+	hr, err := hash.NewReader(gr, objInfo.Size, "", "", actualSize)
 	if err != nil {
-		return err
+		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
 	_, err = z.PutObject(ctx,
 		bucket,
 		objInfo.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
-			VersionID:   objInfo.VersionID,
-			MTime:       objInfo.ModTime,
-			UserDefined: objInfo.UserDefined,
+			VersionID:    objInfo.VersionID,
+			MTime:        objInfo.ModTime,
+			UserDefined:  objInfo.UserDefined,
+			PreserveETag: objInfo.ETag, // Preserve original ETag to ensure same metadata.
+			IndexCB: func() []byte {
+				return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
+			},
 		})
+	if err != nil {
+		err = fmt.Errorf("decommissionObject: PutObject() %w", err)
+	}
 	return err
 }
 
@@ -647,6 +683,33 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		}
 
 		vc, _ := globalBucketVersioningSys.Get(bi.Name)
+
+		// Check if the current bucket has a configured lifecycle policy
+		lc, _ := globalLifecycleSys.Get(bi.Name)
+
+		// Check if bucket is object locked.
+		lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+
+		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
+			if lc == nil {
+				return false
+			}
+			versioned := vc != nil && vc.Versioned(object)
+			objInfo := fi.ToObjectInfo(bucket, object, versioned)
+			action := evalActionFromLifecycle(ctx, *lc, lr, objInfo, false)
+			switch action {
+			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				// Skip this entry.
+				return true
+			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				// Skip this entry.
+				return true
+			}
+			return false
+		}
+
 		decommissionEntry := func(entry metaCacheEntry) {
 			defer func() {
 				<-parallelWorkers
@@ -673,6 +736,13 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bi.Name, version.Name))
 					continue
 				}
+
+				// Apply lifecycle rules on the objects that are expired.
+				if filterLifecycle(bi.Name, version.Name, version) {
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bi.Name, version.Name, version.VersionID))
+					continue
+				}
+
 				// We will skip decommissioning delete markers
 				// with single version, its as good as there
 				// is no data associated with the object.
@@ -680,16 +750,18 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bi.Name, version.Name))
 					continue
 				}
+
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
 						bi.Name,
 						version.Name,
 						ObjectOptions{
-							Versioned:         vc.PrefixEnabled(version.Name),
-							VersionID:         version.VersionID,
-							MTime:             version.ModTime,
-							DeleteReplication: version.ReplicationState,
-							DeleteMarker:      true, // make sure we create a delete marker
+							Versioned:          vc.PrefixEnabled(version.Name),
+							VersionID:          version.VersionID,
+							MTime:              version.ModTime,
+							DeleteReplication:  version.ReplicationState,
+							DeleteMarker:       true, // make sure we create a delete marker
+							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 						})
 					var failure bool
 					if err != nil {
@@ -716,7 +788,8 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						http.Header{},
 						noLock, // all mutations are blocked reads are safe without locks.
 						ObjectOptions{
-							VersionID: version.VersionID,
+							VersionID:    version.VersionID,
+							NoDecryption: true,
 						})
 					if isErrObjectNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
@@ -791,7 +864,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					wg.Add(1)
 					go decommissionEntry(entry)
 				},
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				partial: func(entries metaCacheEntries, _ []error) {
 					entry, ok := entries.resolve(&resolver)
 					if ok {
 						parallelWorkers <- struct{}{}
@@ -912,15 +985,9 @@ func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpace
 	if idx+1 > len(z.serverPools) {
 		return pi, errInvalidArgument
 	}
+
 	info, _ := z.serverPools[idx].StorageInfo(context.Background())
 	info.Backend = z.BackendInfo()
-	for _, disk := range info.Disks {
-		if disk.Healing {
-			return pi, decomError{
-				Err: fmt.Sprintf("%s drive is healing, decommission will not be started", disk.Endpoint),
-			}
-		}
-	}
 
 	usableTotal := int64(GetTotalUsableCapacity(info.Disks, info))
 	usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info))
@@ -984,7 +1051,9 @@ func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (e
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionCancel(idx) {
-		defer z.decommissionCancelers[idx]() // cancel any active thread.
+		if fn := z.decommissionCancelers[idx]; fn != nil {
+			defer fn() // cancel any active thread.
+		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1006,7 +1075,9 @@ func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (e
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionFailed(idx) {
-		defer z.decommissionCancelers[idx]() // cancel any active thread.
+		if fn := z.decommissionCancelers[idx]; fn != nil {
+			defer fn() // cancel any active thread.
+		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1028,7 +1099,9 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionComplete(idx) {
-		defer z.decommissionCancelers[idx]() // cancel any active thread.
+		if fn := z.decommissionCancelers[idx]; fn != nil {
+			defer fn() // cancel any active thread.
+		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1051,6 +1124,13 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return err
 	}
 
+	// Make sure to heal the buckets to ensure the new
+	// pool has the new buckets, this is to avoid
+	// failures later.
+	for _, bucket := range buckets {
+		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
+	}
+
 	decomBuckets := make([]decomBucketInfo, len(buckets))
 	for i := range buckets {
 		decomBuckets[i] = decomBucketInfo{
@@ -1132,7 +1212,7 @@ func auditLogDecom(ctx context.Context, apiName, bucket, object, versionID strin
 		errStr = err.Error()
 	}
 	auditLogInternal(ctx, bucket, object, AuditLogOptions{
-		Trigger:   "decommissioning",
+		Event:     "decommission",
 		APIName:   apiName,
 		VersionID: versionID,
 		Error:     errStr,

cmd/erasure-server-pool.go

@@ -427,8 +427,9 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 	})
 
 	for _, pinfo := range poolObjInfos {
-		// skip all objects from suspended pools for mutating calls.
-		if z.IsSuspended(pinfo.PoolIndex) && opts.Mutate {
+		// skip all objects from suspended pools if asked by the
+		// caller.
+		if z.IsSuspended(pinfo.PoolIndex) && opts.SkipDecommissioned {
 			continue
 		}
 
@@ -461,8 +462,8 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 // The check is skipped if there is only one pool, and 0, nil is always returned in that case.
 func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
 	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
-		NoLock: true,
-		Mutate: true,
+		NoLock:             true,
+		SkipDecommissioned: true,
 	})
 }
 
@@ -486,7 +487,7 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 // if none are found falls back to most available space pool, this function is
 // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
-	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{Mutate: true})
+	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{SkipDecommissioned: true})
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
 	}
@@ -976,13 +977,8 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 }
 
 func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error {
-	for idx, pool := range z.serverPools {
-		if z.IsSuspended(idx) {
-			logger.LogIf(ctx, fmt.Errorf("pool %d is suspended, all writes are suspended", idx+1))
-			continue
-		}
-		_, err := pool.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
-		if err != nil {
+	for _, pool := range z.serverPools {
+		if _, err := pool.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true}); err != nil {
 			return err
 		}
 	}
@@ -1325,7 +1321,10 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 	poolResult.KeyMarker = keyMarker
 	poolResult.Prefix = prefix
 	poolResult.Delimiter = delimiter
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
 			delimiter, maxUploads)
 		if err != nil {
@@ -1350,6 +1349,9 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
 		if err != nil {
 			return "", err
@@ -1392,7 +1394,10 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
@@ -1421,7 +1426,10 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 	if z.SinglePool() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		mi, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return mi, nil
@@ -1450,7 +1458,10 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 	if z.SinglePool() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
@@ -1478,7 +1489,10 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
@@ -1507,7 +1521,10 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
@@ -1652,7 +1669,10 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
 	if z.SinglePool() {
 		buckets, err = z.serverPools[0].ListBuckets(ctx)
 	} else {
-		for _, pool := range z.serverPools {
+		for idx, pool := range z.serverPools {
+			if z.IsSuspended(idx) {
+				continue
+			}
 			buckets, err = pool.ListBuckets(ctx)
 			if err != nil {
 				logger.LogIf(ctx, err)
@@ -1822,7 +1842,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						minDisks:       1,
 						reportNotFound: false,
 						agreed:         loadEntry,
-						partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+						partial: func(entries metaCacheEntries, _ []error) {
 							entry, ok := entries.resolve(&resolver)
 							if !ok {
 								// check if we can get one entry atleast
@@ -1889,7 +1909,7 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 				cancel()
 			}
 		},
-		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+		partial: func(entries metaCacheEntries, _ []error) {
 			entry, ok := entries.resolve(&resolver)
 			if !ok {
 				// check if we can get one entry atleast
@@ -2256,7 +2276,6 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = false
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, err
@@ -2272,7 +2291,8 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true // Avoid transitioning an object from a pool being decommissioned.
+	// Avoid transitioning an object from a pool being decommissioned.
+	opts.SkipDecommissioned = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
@@ -2288,7 +2308,8 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true // Avoid restoring object from a pool being decommissioned.
+	// Avoid restoring object from a pool being decommissioned.
+	opts.SkipDecommissioned = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err

cmd/erasure-sets.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"context"
 	"encoding/binary"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -549,29 +548,13 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 	}
 }
 
-const objectErasureMapKey = "objectErasureMap"
-
 type auditObjectOp struct {
+	Name  string   `json:"name"`
 	Pool  int      `json:"poolId"`
 	Set   int      `json:"setId"`
 	Disks []string `json:"disks"`
 }
 
-type auditObjectErasureMap struct {
-	sync.Map
-}
-
-// Define how to marshal auditObjectErasureMap so it can be
-// printed in the audit webhook notification request.
-func (a *auditObjectErasureMap) MarshalJSON() ([]byte, error) {
-	mapCopy := make(map[string]auditObjectOp)
-	a.Range(func(k, v interface{}) bool {
-		mapCopy[k.(string)] = v.(auditObjectOp)
-		return true
-	})
-	return json.Marshal(mapCopy)
-}
-
 // Add erasure set information to the current context
 func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjects) {
 	if len(logger.AuditTargets()) == 0 {
@@ -579,33 +562,20 @@ func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjec
 	}
 
 	object = decodeDirObject(object)
-
-	var disksEndpoints []string
-	for _, endpoint := range set.getEndpoints() {
+	endpoints := set.getEndpoints()
+	disksEndpoints := make([]string, 0, len(endpoints))
+	for _, endpoint := range endpoints {
 		disksEndpoints = append(disksEndpoints, endpoint.String())
 	}
 
 	op := auditObjectOp{
+		Name:  object,
 		Pool:  set.poolIndex + 1,
 		Set:   set.setIndex + 1,
 		Disks: disksEndpoints,
 	}
 
-	var objectErasureSetTag *auditObjectErasureMap
-	reqInfo := logger.GetReqInfo(ctx)
-	for _, kv := range reqInfo.GetTags() {
-		if kv.Key == objectErasureMapKey {
-			objectErasureSetTag = kv.Val.(*auditObjectErasureMap)
-			break
-		}
-	}
-
-	if objectErasureSetTag == nil {
-		objectErasureSetTag = &auditObjectErasureMap{}
-	}
-
-	objectErasureSetTag.Store(object, op)
-	reqInfo.SetTags(objectErasureMapKey, objectErasureSetTag)
+	logger.GetReqInfo(ctx).AppendTags("objectLocation", op)
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -1195,8 +1165,12 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 		// Do nothing
 		return
 	}
-	infos, _ := getHealDiskInfos(storageDisks, errs)
+	infos, ierrs := getHealDiskInfos(storageDisks, errs)
 	for i := range storageDisks {
+		if ierrs[i] != nil && ierrs[i] != errUnformattedDisk {
+			storageDisks[i] = nil
+			continue
+		}
 		if storageDisks[i] != nil && infos[i].RootDisk {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.

cmd/erasure-single-drive.go

@@ -849,6 +849,11 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
 	}
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	for i, w := range writers {
 		if w == nil {
 			// Make sure to avoid writing to disks which we couldn't complete in erasure.Encode()
@@ -856,7 +861,7 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -1082,6 +1087,11 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		defer lk.Unlock(lkctx.Cancel)
 	}
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -1092,7 +1102,7 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -1321,8 +1331,14 @@ func (es *erasureSingle) DeleteObjects(ctx context.Context, bucket string, objec
 
 func (es *erasureSingle) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	dirPrefix := encodeDirObject(prefix)
-	defer es.disk.Delete(ctx, bucket, dirPrefix, true)
-	return es.disk.Delete(ctx, bucket, prefix, true)
+	defer es.disk.Delete(ctx, bucket, dirPrefix, DeleteOptions{
+		Recursive: true,
+		Force:     true,
+	})
+	return es.disk.Delete(ctx, bucket, prefix, DeleteOptions{
+		Recursive: true,
+		Force:     true,
+	})
 }
 
 // DeleteObject - deletes an object, this call doesn't necessary reply
@@ -1916,7 +1932,10 @@ func (es *erasureSingle) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 			return nil
 		}, index)
 	}
@@ -1954,7 +1973,10 @@ func (es *erasureSingle) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		wait := es.deletedCleanupSleeper.Timer(ctx)
 		if now.Sub(vi.Created) > expiry {
-			disk.Delete(ctx, minioMetaTmpBucket, tmpDir, true)
+			disk.Delete(ctx, minioMetaTmpBucket, tmpDir, DeleteOptions{
+				Recursive: true,
+				Force:     false,
+			})
 		}
 		wait()
 		return nil
@@ -2357,8 +2379,13 @@ func (es *erasureSingle) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	md5hex := r.MD5CurrentHexString()
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	// Add the current part.
-	fi.AddObjectPart(partID, md5hex, n, data.ActualSize())
+	fi.AddObjectPart(partID, md5hex, n, data.ActualSize(), index)
 
 	for i, disk := range onlineDisks {
 		if disk == OfflineDisk {
@@ -2656,6 +2683,7 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 			Number:     part.PartNumber,
 			Size:       currentFI.Parts[partIdx].Size,
 			ActualSize: currentFI.Parts[partIdx].ActualSize,
+			Index:      currentFI.Parts[partIdx].Index,
 		}
 	}
 
@@ -2984,7 +3012,7 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 				minDisks:       1,
 				reportNotFound: false,
 				agreed:         loadEntry,
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				partial: func(entries metaCacheEntries, _ []error) {
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						// check if we can get one entry atleast

cmd/format-erasure.go

@@ -367,7 +367,10 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, false)
+	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 
 	// write to unique file.
 	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatBytes); err != nil {

cmd/gateway-startup-msg.go

@@ -48,12 +48,6 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 
 	// Prints documentation message.
 	printObjectAPIMsg()
-
-	if globalMinioConsolePortAuto && globalBrowserEnabled {
-		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
-			globalMinioConsolePort)
-		logger.Info(color.RedBold(msg))
-	}
 }
 
 // Prints common server startup message. Prints credential, region and browser access.

cmd/global-heal.go

@@ -316,7 +316,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			minDisks:       1,
 			reportNotFound: false,
 			agreed:         healEntry,
-			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+			partial: func(entries metaCacheEntries, _ []error) {
 				entry, ok := entries.resolve(&resolver)
 				if !ok {
 					// check if we can get one entry atleast

cmd/globals.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/minio/console/restapi"
+	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/config"
@@ -173,9 +174,8 @@ var (
 	globalMinioAddr = ""
 
 	// MinIO default port, can be changed through command line.
-	globalMinioPort            = GlobalMinioDefaultPort
-	globalMinioConsolePort     = "13333"
-	globalMinioConsolePortAuto = false
+	globalMinioPort        = GlobalMinioDefaultPort
+	globalMinioConsolePort = "13333"
 
 	// Holds the host that was passed using --address
 	globalMinioHost = ""
@@ -377,6 +377,9 @@ var (
 	// MinIO version unix timestamp
 	globalVersionUnix uint64
 
+	// MinIO client
+	globalMinioClient *minio.Client
+
 	// Add new variable global values here.
 )
 

cmd/handler-utils.go

@@ -27,7 +27,6 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
-	"net/url"
 	"regexp"
 	"strings"
 
@@ -197,16 +196,6 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
 	return nil
 }
 
-// The Query string for the redirect URL the client is
-// redirected on successful upload.
-func getRedirectPostRawQuery(objInfo ObjectInfo) string {
-	redirectValues := make(url.Values)
-	redirectValues.Set("bucket", objInfo.Bucket)
-	redirectValues.Set("key", objInfo.Name)
-	redirectValues.Set("etag", "\""+objInfo.ETag+"\"")
-	return redirectValues.Encode()
-}
-
 // Returns access credentials in the request Authorization header.
 func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
 	cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)

cmd/metacache-server-pool.go

@@ -667,8 +667,11 @@ func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle,
 		action := evalActionFromLifecycle(ctx, lc, lr, objInfo, false)
 		switch action {
 		case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-			fallthrough
+			globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+			// Skip this entry.
+			continue
 		case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+			globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
 			// Skip this entry.
 			continue
 		}

cmd/metacache-set.go

@@ -760,7 +760,7 @@ func (es *erasureSingle) listPathInner(ctx context.Context, o listPathOptions, r
 			case results <- entry:
 			}
 		},
-		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+		partial: func(entries metaCacheEntries, errs []error) {
 			// Results Disagree :-(
 			entry, ok := entries.resolve(&resolver)
 			if ok {
@@ -829,7 +829,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 			case results <- entry:
 			}
 		},
-		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+		partial: func(entries metaCacheEntries, errs []error) {
 			// Results Disagree :-(
 			entry, ok := entries.resolve(&resolver)
 			if ok {
@@ -1145,7 +1145,7 @@ type listPathRawOptions struct {
 	// partial will be called when there is disagreement between disks.
 	// if disk did not return any result, but also haven't errored
 	// the entry will be empty and errs will
-	partial func(entries metaCacheEntries, nAgreed int, errs []error)
+	partial func(entries metaCacheEntries, errs []error)
 
 	// finished will be called when all streams have finished and
 	// more than one disk returned an error.
@@ -1362,7 +1362,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			continue
 		}
 		if opts.partial != nil {
-			opts.partial(topEntries, agree, errs)
+			opts.partial(topEntries, errs)
 		}
 		// Skip the inputs we used.
 		for i, r := range readers {

cmd/metrics-v2.go

@@ -27,6 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
@@ -49,6 +50,7 @@ func init() {
 		getNodeHealthMetrics(),
 		getClusterStorageMetrics(),
 		getClusterTierMetrics(),
+		getKMSMetrics(),
 	}
 
 	peerMetricsGroups = []*MetricsGroup{
@@ -63,6 +65,7 @@ func init() {
 		getILMNodeMetrics(),
 		getScannerNodeMetrics(),
 		getIAMNodeMetrics(),
+		getKMSNodeMetrics(),
 	}
 
 	allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -80,7 +83,6 @@ func init() {
 		getMinioVersionMetrics(),
 		getS3TTFBMetric(),
 	})
-
 	clusterCollector = newMinioClusterCollector(allMetricsGroups)
 }
 
@@ -123,6 +125,7 @@ const (
 	ilmSubsystem              MetricSubsystem = "ilm"
 	scannerSubsystem          MetricSubsystem = "scanner"
 	iamSubsystem              MetricSubsystem = "iam"
+	kmsSubsystem              MetricSubsystem = "kms"
 )
 
 // MetricName are the individual names for the metric.
@@ -188,6 +191,12 @@ const (
 	transitionedBytes    MetricName = "transitioned_bytes"
 	transitionedObjects  MetricName = "transitioned_objects"
 	transitionedVersions MetricName = "transitioned_versions"
+
+	kmsOnline          = "online"
+	kmsRequestsSuccess = "request_success"
+	kmsRequestsError   = "request_error"
+	kmsRequestsFail    = "request_failure"
+	kmsUptime          = "uptime"
 )
 
 const (
@@ -232,6 +241,7 @@ type MetricsGroup struct {
 // to populate new values upon cache invalidation.
 func (g *MetricsGroup) RegisterRead(read func(ctx context.Context) []Metric) {
 	g.metricsCache.Once.Do(func() {
+		g.metricsCache.Relax = true
 		g.metricsCache.TTL = g.cacheInterval
 		g.metricsCache.Update = func() (interface{}, error) {
 			return read(GlobalContext), nil
@@ -1930,6 +1940,107 @@ func getClusterStorageMetrics() *MetricsGroup {
 	return mg
 }
 
+func getKMSNodeMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 10 * time.Second,
+	}
+
+	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
+		objLayer := newObjectLayerFn()
+		// Service not initialized yet
+		if objLayer == nil || globalIsGateway || GlobalKMS == nil {
+			return
+		}
+
+		const (
+			Online  = 1
+			Offline = 0
+		)
+		desc := MetricDescription{
+			Namespace: clusterMetricNamespace,
+			Subsystem: kmsSubsystem,
+			Name:      kmsOnline,
+			Help:      "Reports whether the KMS is online (1) or offline (0)",
+			Type:      gaugeMetric,
+		}
+		_, err := GlobalKMS.Metrics(ctx)
+		if _, ok := kes.IsConnError(err); ok {
+			return []Metric{{
+				Description: desc,
+				Value:       float64(Offline),
+			}}
+		}
+		return []Metric{{
+			Description: desc,
+			Value:       float64(Online),
+		}}
+	})
+	return mg
+}
+
+func getKMSMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 10 * time.Second,
+	}
+
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		objLayer := newObjectLayerFn()
+		// Service not initialized yet
+		if objLayer == nil || globalIsGateway || GlobalKMS == nil {
+			return []Metric{}
+		}
+
+		metrics := make([]Metric, 0, 4)
+		metric, err := GlobalKMS.Metrics(ctx)
+		if err != nil {
+			return metrics
+		}
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsRequestsSuccess,
+				Help:      "Number of KMS requests that succeeded",
+				Type:      counterMetric,
+			},
+			Value: float64(metric.RequestOK),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsRequestsError,
+				Help:      "Number of KMS requests that failed due to some error. (HTTP 4xx status code)",
+				Type:      counterMetric,
+			},
+			Value: float64(metric.RequestErr),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsRequestsFail,
+				Help:      "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)",
+				Type:      counterMetric,
+			},
+			Value: float64(metric.RequestFail),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: clusterMetricNamespace,
+				Subsystem: kmsSubsystem,
+				Name:      kmsUptime,
+				Help:      "The time the KMS has been up and running in seconds.",
+				Type:      counterMetric,
+			},
+			Value: metric.UpTime.Seconds(),
+		})
+
+		return metrics
+	})
+	return mg
+}
+
 type minioClusterCollector struct {
 	metricsGroups []*MetricsGroup
 	desc          *prometheus.Desc

cmd/naughty-disk_test.go

@@ -221,11 +221,11 @@ func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string
 	return d.disk.CheckParts(ctx, volume, path, fi)
 }
 
-func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
+func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.Delete(ctx, volume, path, recursive)
+	return d.disk.Delete(ctx, volume, path, deleteOpts)
 }
 
 func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error {

cmd/notification.go

@@ -1512,17 +1512,15 @@ func (sys *NotificationSys) Netperf(ctx context.Context, duration time.Duration)
 	return results
 }
 
-// Speedtest run GET/PUT tests at input concurrency for requested object size,
+// SpeedTest run GET/PUT tests at input concurrency for requested object size,
 // optionally you can extend the tests longer with time.Duration.
-func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
-	concurrent int, duration time.Duration, storageClass string,
-) []SpeedtestResult {
+func (sys *NotificationSys) SpeedTest(ctx context.Context, sopts speedTestOpts) []SpeedTestResult {
 	length := len(sys.allPeerClients)
 	if length == 0 {
 		// For single node erasure setup.
 		length = 1
 	}
-	results := make([]SpeedtestResult, length)
+	results := make([]SpeedTestResult, length)
 
 	scheme := "http"
 	if globalIsTLS {
@@ -1537,8 +1535,7 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
 		wg.Add(1)
 		go func(index int) {
 			defer wg.Done()
-			r, err := sys.peerClients[index].Speedtest(ctx, size,
-				concurrent, duration, storageClass)
+			r, err := sys.peerClients[index].SpeedTest(ctx, sopts)
 			u := &url.URL{
 				Scheme: scheme,
 				Host:   sys.peerClients[index].host.String(),
@@ -1555,7 +1552,7 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		r, err := selfSpeedtest(ctx, size, concurrent, duration, storageClass)
+		r, err := selfSpeedTest(ctx, sopts)
 		u := &url.URL{
 			Scheme: scheme,
 			Host:   globalLocalNodeName,

cmd/object-api-datatypes.go

@@ -262,6 +262,7 @@ func (o ObjectInfo) tierStats() tierStats {
 type ReplicateObjectInfo struct {
 	ObjectInfo
 	OpType            replication.Type
+	EventType         string
 	RetryCount        uint32
 	ResetID           string
 	Dsc               ReplicateDecision

cmd/object-api-interface.go

@@ -59,6 +59,8 @@ type ObjectOptions struct {
 	Transition        TransitionOptions
 	Expiration        ExpirationOptions
 
+	NoDecryption                        bool      // indicates if the stream must be decrypted.
+	PreserveETag                        string    // preserves this etag during a PUT call.
 	NoLock                              bool      // indicates to lower layers if the caller is expecting to hold locks.
 	ProxyRequest                        bool      // only set for GET/HEAD in active-active replication scenario
 	ProxyHeaderSet                      bool      // only set for GET/HEAD in active-active replication scenario
@@ -73,11 +75,16 @@ type ObjectOptions struct {
 	// Use the maximum parity (N/2), used when saving server configuration files
 	MaxParity bool
 
-	// Mutate set to 'true' if the call is namespace mutation call
-	Mutate        bool
-	WalkAscending bool // return Walk results in ascending order of versions
+	// SkipDecommissioned set to 'true' if the call requires skipping the pool being decommissioned.
+	// mainly set for certain WRITE operations.
+	SkipDecommissioned bool
 
+	WalkAscending   bool                     // return Walk results in ascending order of versions
 	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
+
+	// IndexCB will return any index created but the compression.
+	// Object must have been read at this point.
+	IndexCB func() []byte
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/object-api-options.go

@@ -74,6 +74,7 @@ func getDefaultOpts(header http.Header, copySource bool, metadata map[string]str
 	if _, ok := header[xhttp.MinIOSourceReplicationRequest]; ok {
 		opts.ReplicationRequest = true
 	}
+	opts.Speedtest = header.Get(globalObjectPerfUserMetadata) != ""
 	return
 }
 

cmd/object-api-utils.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/hmac"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -43,12 +44,15 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/fips"
 	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/trie"
 	"github.com/minio/pkg/wildcard"
+	"github.com/minio/sio"
 )
 
 const (
@@ -70,6 +74,10 @@ const (
 	compReadAheadBuffers = 5
 	// Size of each buffer.
 	compReadAheadBufSize = 1 << 20
+	// Pad Encrypted+Compressed files to a multiple of this.
+	compPadEncrypted = 256
+	// Disable compressed file indices below this size
+	compMinIndexSize = 8 << 20
 )
 
 // isMinioBucket returns true if given bucket is a MinIO internal
@@ -432,11 +440,7 @@ func isCompressible(header http.Header, object string) bool {
 	cfg := globalCompressConfig
 	globalCompressConfigMu.Unlock()
 
-	_, ok := crypto.IsRequested(header)
-	if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
-		return false
-	}
-	return true
+	return !excludeForCompression(header, object, cfg)
 }
 
 // Eliminate the non-compressible objects.
@@ -447,6 +451,10 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
 		return true
 	}
 
+	if crypto.Requested(header) && !cfg.AllowEncrypted {
+		return true
+	}
+
 	// We strictly disable compression for standard extensions/content-types (`compressed`).
 	if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
 		return true
@@ -513,12 +521,12 @@ func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec {
 // Returns the compressed offset which should be skipped.
 // If encrypted offsets are adjusted for encrypted block headers/trailers.
 // Since de-compression is after decryption encryption overhead is only added to compressedOffset.
-func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset int64, partSkip int64, firstPart int) {
+func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]byte, error)) (compressedOffset int64, partSkip int64, firstPart int, decryptSkip int64, seqNum uint32) {
 	var skipLength int64
 	var cumulativeActualSize int64
 	var firstPartIdx int
-	if len(objectInfo.Parts) > 0 {
-		for i, part := range objectInfo.Parts {
+	if len(oi.Parts) > 0 {
+		for i, part := range oi.Parts {
 			cumulativeActualSize += part.ActualSize
 			if cumulativeActualSize <= offset {
 				compressedOffset += part.Size
@@ -529,8 +537,52 @@ func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset
 			}
 		}
 	}
+	partSkip = offset - skipLength
 
-	return compressedOffset, offset - skipLength, firstPartIdx
+	// Load index and skip more if feasible.
+	if partSkip > 0 && len(oi.Parts) > firstPartIdx && len(oi.Parts[firstPartIdx].Index) > 0 {
+		_, isEncrypted := crypto.IsEncrypted(oi.UserDefined)
+		if isEncrypted {
+			dec, err := decrypt(oi.Parts[firstPartIdx].Index)
+			if err == nil {
+				// Load Index
+				var idx s2.Index
+				_, err := idx.Load(s2.RestoreIndexHeaders(dec))
+
+				// Find compressed/uncompressed offsets of our partskip
+				compOff, uCompOff, err2 := idx.Find(partSkip)
+
+				if err == nil && err2 == nil && compOff > 0 {
+					// Encrypted.
+					const sseDAREEncPackageBlockSize = SSEDAREPackageBlockSize + SSEDAREPackageMetaSize
+					// Number of full blocks in skipped area
+					seqNum = uint32(compOff / SSEDAREPackageBlockSize)
+					// Skip this many inside a decrypted block to get to compression block start
+					decryptSkip = compOff % SSEDAREPackageBlockSize
+					// Skip this number of full blocks.
+					skipEnc := compOff / SSEDAREPackageBlockSize
+					skipEnc *= sseDAREEncPackageBlockSize
+					compressedOffset += skipEnc
+					// Skip this number of uncompressed bytes.
+					partSkip -= uCompOff
+				}
+			}
+		} else {
+			// Not encrypted
+			var idx s2.Index
+			_, err := idx.Load(s2.RestoreIndexHeaders(oi.Parts[firstPartIdx].Index))
+
+			// Find compressed/uncompressed offsets of our partskip
+			compOff, uCompOff, err2 := idx.Find(partSkip)
+
+			if err == nil && err2 == nil && compOff > 0 {
+				compressedOffset += compOff
+				partSkip -= uCompOff
+			}
+		}
+	}
+
+	return compressedOffset, partSkip, firstPartIdx, decryptSkip, seqNum
 }
 
 // GetObjectReader is a type that wraps a reader with a lock to
@@ -595,8 +647,9 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 		return nil, 0, 0, err
 	}
 
-	// if object is encrypted and it is a restore request, fetch content without decrypting.
-	if opts.Transition.RestoreRequest != nil {
+	// if object is encrypted and it is a restore request or if NoDecryption
+	// was requested, fetch content without decrypting.
+	if opts.Transition.RestoreRequest != nil || opts.NoDecryption {
 		isEncrypted = false
 		isCompressed = false
 	}
@@ -618,6 +671,8 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 		if err != nil {
 			return nil, 0, 0, err
 		}
+		var decryptSkip int64
+		var seqNum uint32
 
 		off, length = int64(0), oi.Size
 		decOff, decLength := int64(0), actualSize
@@ -626,10 +681,14 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 			if err != nil {
 				return nil, 0, 0, err
 			}
-
+			decrypt := func(b []byte) ([]byte, error) {
+				return b, nil
+			}
+			if isEncrypted {
+				decrypt = oi.compressionIndexDecrypt
+			}
 			// In case of range based queries on multiparts, the offset and length are reduced.
-			off, decOff, firstPart = getCompressedOffsets(oi, off)
-
+			off, decOff, firstPart, decryptSkip, seqNum = getCompressedOffsets(oi, off, decrypt)
 			decLength = length
 			length = oi.Size - off
 			// For negative length we read everything.
@@ -646,7 +705,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 			if isEncrypted {
 				copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
 				// Attach decrypter on inputReader
-				inputReader, err = DecryptBlocksRequestR(inputReader, h, 0, firstPart, oi, copySource)
+				inputReader, err = DecryptBlocksRequestR(inputReader, h, seqNum, firstPart, oi, copySource)
 				if err != nil {
 					// Call the cleanup funcs
 					for i := len(cFns) - 1; i >= 0; i-- {
@@ -654,10 +713,18 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions) (
 					}
 					return nil, err
 				}
+				if decryptSkip > 0 {
+					inputReader = ioutil.NewSkipReader(inputReader, decryptSkip)
+				}
 				oi.Size = decLength
 			}
 			// Decompression reader.
-			s2Reader := s2.NewReader(inputReader)
+			var dopts []s2.ReaderOption
+			if off > 0 {
+				// We are not starting at the beginning, so ignore stream identifiers.
+				dopts = append(dopts, s2.ReaderIgnoreStreamIdentifier())
+			}
+			s2Reader := s2.NewReader(inputReader, dopts...)
 			// Apply the skipLen and limit on the decompressed stream.
 			if decOff > 0 {
 				if err = s2Reader.Skip(decOff); err != nil {
@@ -778,6 +845,41 @@ func (g *GetObjectReader) Close() error {
 	return nil
 }
 
+func compressionIndexEncrypter(key crypto.ObjectKey, input func() []byte) func() []byte {
+	var data []byte
+	var fetched bool
+	return func() []byte {
+		if !fetched {
+			data = input()
+			fetched = true
+		}
+		if len(data) == 0 {
+			return data
+		}
+		var buffer bytes.Buffer
+		mac := hmac.New(sha256.New, key[:])
+		mac.Write([]byte("compression-index"))
+		if _, err := sio.Encrypt(&buffer, bytes.NewReader(data), sio.Config{Key: mac.Sum(nil), CipherSuites: fips.DARECiphers()}); err != nil {
+			logger.CriticalIf(context.Background(), errors.New("unable to encrypt compression index using object key"))
+		}
+		return buffer.Bytes()
+	}
+}
+
+func (o *ObjectInfo) compressionIndexDecrypt(input []byte) ([]byte, error) {
+	if len(input) == 0 {
+		return input, nil
+	}
+
+	key, err := decryptObjectInfo(nil, o.Bucket, o.Name, o.UserDefined)
+	if err != nil {
+		return nil, err
+	}
+	mac := hmac.New(sha256.New, key)
+	mac.Write([]byte("compression-index"))
+	return sio.DecryptBuffer(nil, input, sio.Config{Key: mac.Sum(nil), CipherSuites: fips.DARECiphers()})
+}
+
 // SealMD5CurrFn seals md5sum with object encryption key and returns sealed
 // md5sum
 type SealMD5CurrFn func([]byte) []byte
@@ -888,11 +990,20 @@ func init() {
 // input 'on' is always recommended such that this function works
 // properly, because we do not wish to create an object even if
 // client closed the stream prematurely.
-func newS2CompressReader(r io.Reader, on int64) io.ReadCloser {
+func newS2CompressReader(r io.Reader, on int64, encrypted bool) (rc io.ReadCloser, idx func() []byte) {
 	pr, pw := io.Pipe()
 	// Copy input to compressor
+	opts := compressOpts
+	if encrypted {
+		// The values used for padding are not a security concern,
+		// but we choose pseudo-random numbers instead of just zeros.
+		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
+		opts = append([]s2.WriterOption{s2.WriterPadding(compPadEncrypted), s2.WriterPaddingSrc(rng)}, compressOpts...)
+	}
+	comp := s2.NewWriter(pw, opts...)
+	indexCh := make(chan []byte, 1)
 	go func() {
-		comp := s2.NewWriter(pw, compressOpts...)
+		defer close(indexCh)
 		cn, err := io.Copy(comp, r)
 		if err != nil {
 			comp.Close()
@@ -907,9 +1018,25 @@ func newS2CompressReader(r io.Reader, on int64) io.ReadCloser {
 			return
 		}
 		// Close the stream.
+		// If more than compMinIndexSize was written, generate index.
+		if cn > compMinIndexSize {
+			idx, err := comp.CloseIndex()
+			idx = s2.RemoveIndexHeaders(idx)
+			indexCh <- idx
+			pw.CloseWithError(err)
+			return
+		}
 		pw.CloseWithError(comp.Close())
 	}()
-	return pr
+	var gotIdx []byte
+	return pr, func() []byte {
+		if gotIdx != nil {
+			return gotIdx
+		}
+		// Will get index or nil if closed.
+		gotIdx = <-indexCh
+		return gotIdx
+	}
 }
 
 // compressSelfTest performs a self-test to ensure that compression
@@ -933,7 +1060,7 @@ func compressSelfTest() {
 		}
 	}
 	const skip = 2<<20 + 511
-	r := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)))
+	r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)), true)
 	b, err := io.ReadAll(r)
 	failOnErr(err)
 	failOnErr(r.Close())

cmd/object-api-utils_test.go

@@ -593,7 +593,7 @@ func TestGetCompressedOffsets(t *testing.T) {
 		},
 	}
 	for i, test := range testCases {
-		startOffset, snappyStartOffset, firstPart := getCompressedOffsets(test.objInfo, test.offset)
+		startOffset, snappyStartOffset, firstPart, _, _ := getCompressedOffsets(test.objInfo, test.offset, nil)
 		if startOffset != test.startOffset {
 			t.Errorf("Test %d - expected startOffset %d but received %d",
 				i, test.startOffset, startOffset)
@@ -611,19 +611,20 @@ func TestGetCompressedOffsets(t *testing.T) {
 
 func TestS2CompressReader(t *testing.T) {
 	tests := []struct {
-		name string
-		data []byte
+		name    string
+		data    []byte
+		wantIdx bool
 	}{
 		{name: "empty", data: nil},
-		{name: "small", data: []byte("hello, world")},
-		{name: "large", data: bytes.Repeat([]byte("hello, world"), 1000)},
+		{name: "small", data: []byte("hello, world!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")},
+		{name: "large", data: bytes.Repeat([]byte("hello, world"), 1000000), wantIdx: true},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case
 
-			r := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)))
+			r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)), false)
 			defer r.Close()
 
 			var rdrBuf bytes.Buffer
@@ -631,7 +632,26 @@ func TestS2CompressReader(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-
+			r.Close()
+			idx := idxCB()
+			if !tt.wantIdx && len(idx) > 0 {
+				t.Errorf("index returned above threshold")
+			}
+			if tt.wantIdx {
+				if idx == nil {
+					t.Errorf("no index returned")
+				}
+				var index s2.Index
+				_, err = index.Load(s2.RestoreIndexHeaders(idx))
+				if err != nil {
+					t.Errorf("error loading index: %v", err)
+				}
+				t.Log("size:", len(idx))
+				t.Log(string(index.JSON()))
+				if index.TotalUncompressed != int64(len(tt.data)) {
+					t.Errorf("Expected size %d, got %d", len(tt.data), index.TotalUncompressed)
+				}
+			}
 			var stdBuf bytes.Buffer
 			w := s2.NewWriter(&stdBuf)
 			_, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf)

cmd/object-handlers.go

@@ -83,6 +83,9 @@ const (
 	encryptBufferThreshold = 1 << 20
 	// add an input buffer of this size.
 	encryptBufferSize = 1 << 20
+
+	// minCompressibleSize is the minimum size at which we enable compression.
+	minCompressibleSize = 4096
 )
 
 // setHeadGetRespHeaders - set any requested parameters as response headers.
@@ -116,7 +119,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
@@ -327,7 +330,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
@@ -608,7 +611,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest))
 		return
 	}
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
@@ -948,7 +951,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
+	if crypto.Requested(r.Header) {
 		if globalIsGateway {
 			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -1148,7 +1151,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Check if either the source is encrypted or the destination will be encrypted.
-	_, objectEncryption := crypto.IsRequested(r.Header)
+	objectEncryption := crypto.Requested(r.Header)
 	objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined)
 
 	var compressMetadata map[string]string
@@ -1156,6 +1159,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// Pass the decompressed stream to such calls.
 	isDstCompressed := objectAPI.IsCompressionSupported() &&
 		isCompressible(r.Header, dstObject) &&
+		length > minCompressibleSize &&
 		!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) && !cpSrcDstSame && !objectEncryption
 	if isDstCompressed {
 		compressMetadata = make(map[string]string, 2)
@@ -1164,7 +1168,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
 
 		reader = etag.NewReader(reader, nil)
-		s2c := newS2CompressReader(reader, actualSize)
+		wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
+		s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption)
+		dstOpts.IndexCB = cb
 		defer s2c.Close()
 		reader = etag.Wrap(s2c, reader)
 		length = -1
@@ -1308,6 +1314,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
+				if dstOpts.IndexCB != nil {
+					dstOpts.IndexCB = compressionIndexEncrypter(objEncKey, dstOpts.IndexCB)
+				}
 			}
 		}
 	}
@@ -1565,7 +1574,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
+	if crypto.Requested(r.Header) {
 		if globalIsGateway {
 			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -1715,7 +1724,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	})
 
 	actualSize := size
-	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
+	var idxCb func() []byte
+	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > minCompressibleSize {
 		// Storing the compression metadata.
 		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
@@ -1727,8 +1737,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		// Set compression metrics.
-		s2c := newS2CompressReader(actualReader, actualSize)
+		var s2c io.ReadCloser
+		wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
+		s2c, idxCb = newS2CompressReader(actualReader, actualSize, wantEncryption)
 		defer s2c.Close()
+
 		reader = etag.Wrap(s2c, actualReader)
 		size = -1   // Since compressed size is un-predictable.
 		md5hex = "" // Do not try to verify the content.
@@ -1751,6 +1764,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	opts.IndexCB = idxCb
 
 	if api.CacheAPI() != nil {
 		putObject = api.CacheAPI().PutObject
@@ -1784,7 +1798,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 	var objectEncryptionKey crypto.ObjectKey
 	if objectAPI.IsEncryptionSupported() {
-		if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if crypto.Requested(r.Header) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
 				return
@@ -1813,6 +1827,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
+			if opts.IndexCB != nil {
+				opts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, opts.IndexCB)
+			}
 		}
 	}
 
@@ -1914,7 +1931,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
+	if crypto.Requested(r.Header) {
 		if globalIsGateway {
 			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -2061,7 +2078,8 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		}
 
 		actualSize := size
-		if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
+		var idxCb func() []byte
+		if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > minCompressibleSize {
 			// Storing the compression metadata.
 			metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 			metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
@@ -2072,8 +2090,10 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			}
 
 			// Set compression metrics.
-			s2c := newS2CompressReader(actualReader, actualSize)
+			wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
+			s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption)
 			defer s2c.Close()
+			idxCb = cb
 			reader = etag.Wrap(s2c, actualReader)
 			size = -1 // Since compressed size is un-predictable.
 		}
@@ -2100,6 +2120,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return err
 		}
 		opts.MTime = info.ModTime()
+		opts.IndexCB = idxCb
 
 		retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
 		if s3err == ErrNone && retentionMode.Valid() {
@@ -2126,7 +2147,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 
 		var objectEncryptionKey crypto.ObjectKey
 		if objectAPI.IsEncryptionSupported() {
-			if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+			if crypto.Requested(r.Header) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 				if crypto.SSECopy.IsRequested(r.Header) {
 					return errInvalidEncryptionParameters
 				}
@@ -2153,6 +2174,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 					return err
 				}
 			}
+			if opts.IndexCB != nil {
+				opts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, opts.IndexCB)
+			}
 		}
 
 		// Ensure that metadata does not contain sensitive information
@@ -2213,7 +2237,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
+	if crypto.Requested(r.Header) {
 		if globalIsGateway {
 			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -2258,7 +2282,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	encMetadata := map[string]string{}
 
 	if objectAPI.IsEncryptionSupported() {
-		if _, ok := crypto.IsRequested(r.Header); ok {
+		if crypto.Requested(r.Header) {
 			if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -2356,7 +2380,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -2571,8 +2595,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	// Read compression metadata preserved in the init multipart for the decision.
 	_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
 	// Compress only if the compression is enabled during initial multipart.
+	var idxCb func() []byte
 	if isCompressed {
-		s2c := newS2CompressReader(reader, actualPartSize)
+		wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
+		s2c, cb := newS2CompressReader(reader, actualPartSize, wantEncryption)
+		idxCb = cb
 		defer s2c.Close()
 		reader = etag.Wrap(s2c, reader)
 		length = -1
@@ -2589,6 +2616,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	dstOpts.IndexCB = idxCb
 
 	rawReader := srcInfo.Reader
 	pReader := NewPutObjReader(rawReader)
@@ -2643,6 +2671,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		if dstOpts.IndexCB != nil {
+			dstOpts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, dstOpts.IndexCB)
+		}
 	}
 
 	srcInfo.PutObjReader = pReader
@@ -2682,7 +2713,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
+	if crypto.Requested(r.Header) {
 		if globalIsGateway {
 			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -2821,6 +2852,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	// Read compression metadata preserved in the init multipart for the decision.
 	_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
 
+	var idxCb func() []byte
 	if objectAPI.IsCompressionSupported() && isCompressed {
 		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
@@ -2829,7 +2861,9 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		}
 
 		// Set compression metrics.
-		s2c := newS2CompressReader(actualReader, actualSize)
+		wantEncryption := objectAPI.IsEncryptionSupported() && crypto.Requested(r.Header)
+		s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption)
+		idxCb = cb
 		defer s2c.Close()
 		reader = etag.Wrap(s2c, actualReader)
 		size = -1   // Since compressed size is un-predictable.
@@ -2904,7 +2938,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		if idxCb != nil {
+			idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb)
+		}
 	}
+	opts.IndexCB = idxCb
 
 	putObjectPart := objectAPI.PutObjectPart
 	if api.CacheAPI() != nil {
@@ -3534,7 +3572,8 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 				DeleteMarker:          objInfo.DeleteMarker,
 				ReplicationState:      objInfo.getReplicationState(dsc.String(), opts.VersionID, false),
 			},
-			Bucket: bucket,
+			Bucket:    bucket,
+			EventType: ReplicateIncomingDelete,
 		}
 		scheduleReplicationDelete(ctx, dobj, objectAPI)
 	}

cmd/peer-rest-client.go

@@ -803,26 +803,25 @@ func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan Metric
 	return ch, nil
 }
 
-func (client *peerRESTClient) Speedtest(ctx context.Context, size,
-	concurrent int, duration time.Duration, storageClass string,
-) (SpeedtestResult, error) {
+func (client *peerRESTClient) SpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, error) {
 	values := make(url.Values)
-	values.Set(peerRESTSize, strconv.Itoa(size))
-	values.Set(peerRESTConcurrent, strconv.Itoa(concurrent))
-	values.Set(peerRESTDuration, duration.String())
-	values.Set(peerRESTStorageClass, storageClass)
+	values.Set(peerRESTSize, strconv.Itoa(opts.objectSize))
+	values.Set(peerRESTConcurrent, strconv.Itoa(opts.concurrency))
+	values.Set(peerRESTDuration, opts.duration.String())
+	values.Set(peerRESTStorageClass, opts.storageClass)
+	values.Set(peerRESTBucket, opts.bucketName)
 
-	respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedtest, values, nil, -1)
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedTest, values, nil, -1)
 	if err != nil {
-		return SpeedtestResult{}, err
+		return SpeedTestResult{}, err
 	}
 	defer http.DrainBody(respBody)
 	waitReader, err := waitForHTTPResponse(respBody)
 	if err != nil {
-		return SpeedtestResult{}, err
+		return SpeedTestResult{}, err
 	}
 
-	var result SpeedtestResult
+	var result SpeedTestResult
 	err = gob.NewDecoder(waitReader).Decode(&result)
 	if err != nil {
 		return result, err

cmd/peer-rest-common.go

@@ -63,7 +63,7 @@ const (
 	peerRESTMethodUpdateMetacacheListing      = "/updatemetacache"
 	peerRESTMethodGetPeerMetrics              = "/peermetrics"
 	peerRESTMethodLoadTransitionTierConfig    = "/loadtransitiontierconfig"
-	peerRESTMethodSpeedtest                   = "/speedtest"
+	peerRESTMethodSpeedTest                   = "/speedtest"
 	peerRESTMethodDriveSpeedTest              = "/drivespeedtest"
 	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
 	peerRESTMethodReloadPoolMeta              = "/reloadpoolmeta"

cmd/peer-rest-server.go

@@ -1125,7 +1125,7 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
 		return
@@ -1141,6 +1141,7 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
 	storageClass := r.Form.Get(peerRESTStorageClass)
+	bucketName := r.Form.Get(peerRESTBucket)
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1159,7 +1160,13 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
 
 	done := keepHTTPResponseAlive(w)
 
-	result, err := selfSpeedtest(r.Context(), size, concurrent, duration, storageClass)
+	result, err := selfSpeedTest(r.Context(), speedTestOpts{
+		objectSize:   size,
+		concurrency:  concurrent,
+		duration:     duration,
+		storageClass: storageClass,
+		bucketName:   bucketName,
+	})
 	if err != nil {
 		result.Error = err.Error()
 	}
@@ -1312,7 +1319,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(httpTraceHdrs(server.UpdateMetacacheListingHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(httpTraceHdrs(server.GetPeerMetrics))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(httpTraceHdrs(server.SpeedTestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.NetSpeedTestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))

cmd/perf-tests.go

@@ -30,15 +30,13 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/google/uuid"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/pkg/randreader"
 )
 
-// SpeedtestResult return value of the speedtest function
-type SpeedtestResult struct {
+// SpeedTestResult return value of the speedtest function
+type SpeedTestResult struct {
 	Endpoint  string
 	Uploads   uint64
 	Downloads uint64
@@ -50,10 +48,10 @@ func newRandomReader(size int) io.Reader {
 }
 
 // Runs the speedtest on local MinIO process.
-func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
+func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return SpeedtestResult{}, errServerNotInitialized
+		return SpeedTestResult{}, errServerNotInitialized
 	}
 
 	var errOnce sync.Once
@@ -62,48 +60,34 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 	var totalBytesWritten uint64
 	var totalBytesRead uint64
 
-	region := globalSite.Region
-	if region == "" {
-		region = "us-east-1"
-	}
-
-	client, err := minio.New(globalLocalNodeName, &minio.Options{
-		Creds:     credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
-		Secure:    globalIsTLS,
-		Transport: globalProxyTransport,
-		Region:    region,
-	})
-	if err != nil {
-		return SpeedtestResult{}, err
-	}
-
-	objCountPerThread := make([]uint64, concurrent)
+	objCountPerThread := make([]uint64, opts.concurrency)
 
 	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
 	defer uploadsCancel()
 
 	go func() {
-		time.Sleep(duration)
+		time.Sleep(opts.duration)
 		uploadsCancel()
 	}()
 
-	objNamePrefix := uuid.New().String() + SlashSeparator
+	objNamePrefix := pathJoin(speedTest, mustGetUUID())
 
 	userMetadata := make(map[string]string)
-	userMetadata[globalObjectPerfUserMetadata] = "true"
+	userMetadata[globalObjectPerfUserMetadata] = "true" // Bypass S3 API freeze
+	popts := minio.PutObjectOptions{
+		UserMetadata:         userMetadata,
+		DisableContentSha256: true,
+		DisableMultipart:     true,
+	}
 
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
+	wg.Add(opts.concurrency)
+	for i := 0; i < opts.concurrency; i++ {
 		go func(i int) {
 			defer wg.Done()
 			for {
-				reader := newRandomReader(size)
-				tmpObjName := fmt.Sprintf("%s%d.%d", objNamePrefix, i, objCountPerThread[i])
-				info, err := client.PutObject(uploadsCtx, globalObjectPerfBucket, tmpObjName, reader, int64(size), minio.PutObjectOptions{
-					UserMetadata:         userMetadata,
-					DisableContentSha256: true,
-					DisableMultipart:     true,
-				}) // Bypass S3 API freeze
+				reader := newRandomReader(opts.objectSize)
+				tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, objCountPerThread[i]))
+				info, err := globalMinioClient.PutObject(uploadsCtx, opts.bucketName, tmpObjName, reader, int64(opts.objectSize), popts)
 				if err != nil {
 					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
 						errOnce.Do(func() {
@@ -122,18 +106,21 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 
 	// We already saw write failures, no need to proceed into read's
 	if retError != "" {
-		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+		return SpeedTestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
 	}
 
 	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
 	defer downloadsCancel()
 	go func() {
-		time.Sleep(duration)
+		time.Sleep(opts.duration)
 		downloadsCancel()
 	}()
 
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
+	gopts := minio.GetObjectOptions{}
+	gopts.Set(globalObjectPerfUserMetadata, "true") // Bypass S3 API freeze
+
+	wg.Add(opts.concurrency)
+	for i := 0; i < opts.concurrency; i++ {
 		go func(i int) {
 			defer wg.Done()
 			var j uint64
@@ -144,9 +131,8 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 				if objCountPerThread[i] == j {
 					j = 0
 				}
-				opts := minio.GetObjectOptions{}
-				opts.Set(globalObjectPerfUserMetadata, "true") // Bypass S3 API freeze
-				r, err := client.GetObject(downloadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s%d.%d", objNamePrefix, i, j), opts)
+				tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, j))
+				r, err := globalMinioClient.GetObject(downloadsCtx, opts.bucketName, tmpObjName, gopts)
 				if err != nil {
 					errResp, ok := err.(minio.ErrorResponse)
 					if ok && errResp.StatusCode == http.StatusNotFound {
@@ -183,7 +169,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 	}
 	wg.Wait()
 
-	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+	return SpeedTestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
 }
 
 // To collect RX stats during "mc support perf net"

cmd/post-policy_test.go

@@ -447,7 +447,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 	targetObj := keyName + "/upload.txt"
 
 	// The url of success_action_redirect field
-	redirectURL, err := url.Parse("http://www.google.com")
+	redirectURL, err := url.Parse("http://www.google.com?query=value")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -499,7 +499,11 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 		t.Error("Unexpected error: ", err)
 	}
 
-	redirectURL.RawQuery = getRedirectPostRawQuery(info)
+	v := redirectURL.Query()
+	v.Add("bucket", info.Bucket)
+	v.Add("key", info.Name)
+	v.Add("etag", "\""+info.ETag+"\"")
+	redirectURL.RawQuery = v.Encode()
 	expectedLocation := redirectURL.String()
 
 	// Check the new location url

cmd/s3-zip-handlers.go

@@ -66,7 +66,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
@@ -357,7 +357,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest))
 		return
 	}
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}

cmd/server-main.go

@@ -35,6 +35,8 @@ import (
 	"time"
 
 	"github.com/minio/cli"
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/color"
@@ -588,7 +590,8 @@ func serverMain(ctx *cli.Context) {
 		initBackgroundTransition(GlobalContext, newObject)
 
 		go func() {
-			if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
+			err := globalTierConfigMgr.Init(GlobalContext, newObject)
+			if err != nil {
 				logger.LogIf(GlobalContext, err)
 			}
 
@@ -640,6 +643,18 @@ func serverMain(ctx *cli.Context) {
 		printStartupMessage(getAPIEndpoints(), err)
 	}()
 
+	region := globalSite.Region
+	if region == "" {
+		region = "us-east-1"
+	}
+	globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{
+		Creds:     credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
+		Secure:    globalIsTLS,
+		Transport: globalProxyTransport,
+		Region:    region,
+	})
+	logger.FatalIf(err, "Unable to initialize MinIO client")
+
 	if serverDebugLog {
 		logger.Info("== DEBUG Mode enabled ==")
 		logger.Info("Currently set environment settings:")

cmd/server-startup-msg.go

@@ -142,10 +142,6 @@ func printServerCommonMsg(apiEndpoints []string) {
 	}
 	printEventNotifiers()
 
-	if globalMinioConsolePortAuto && globalBrowserEnabled {
-		logger.Info(color.RedBold("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
-			globalMinioConsolePort))
-	}
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
 		logger.Info(color.Blue("Console: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))

cmd/site-replication.go

@@ -2165,6 +2165,9 @@ func (c *SiteReplicationSys) RemoveRemoteTargetsForEndpoint(ctx context.Context,
 	for _, b := range buckets {
 		config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, b.Name)
 		if err != nil {
+			if errors.Is(err, BucketReplicationConfigNotFound{Bucket: b.Name}) {
+				continue
+			}
 			return err
 		}
 		var nRules []sreplication.Rule
@@ -2190,6 +2193,9 @@ func (c *SiteReplicationSys) RemoveRemoteTargetsForEndpoint(ctx context.Context,
 	}
 	for arn, t := range m {
 		if err := globalBucketTargetSys.RemoveTarget(ctx, t.SourceBucket, arn); err != nil {
+			if errors.Is(err, BucketRemoteTargetNotFound{Bucket: t.SourceBucket}) {
+				continue
+			}
 			return err
 		}
 	}
@@ -2808,7 +2814,7 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 				if bi, ok := sris[dIdx].Buckets[s.Bucket]; ok {
 					hasBucket = !bi.CreatedAt.Equal(timeSentinel)
 				}
-				quotaCfgSet := hasBucket && *quotaCfgs[i] != madmin.BucketQuota{}
+				quotaCfgSet := hasBucket && quotaCfgs[i] != nil && *quotaCfgs[i] != madmin.BucketQuota{}
 				ss := madmin.SRBucketStatsSummary{
 					DeploymentID:             s.DeploymentID,
 					HasBucket:                hasBucket,
@@ -3550,20 +3556,13 @@ type srStatusInfo struct {
 }
 
 func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer) error {
-	buckets, err := c.listBuckets(ctx)
+	info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
+		Buckets: true,
+	})
 	if err != nil {
 		return err
 	}
-	for _, bi := range buckets {
-		bucket := bi.Name
-		info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
-			Entity:      madmin.SRBucketEntity,
-			EntityValue: bucket,
-		})
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
+	for bucket := range info.BucketStats {
 		c.healCreateMissingBucket(ctx, objAPI, bucket, info)
 		c.healVersioningMetadata(ctx, objAPI, bucket, info)
 		c.healOLockConfigMetadata(ctx, objAPI, bucket, info)

cmd/speedtest.go

@@ -28,12 +28,16 @@ import (
 	"github.com/minio/madmin-go"
 )
 
+const speedTest = "speedtest"
+
 type speedTestOpts struct {
-	throughputSize   int
+	objectSize       int
 	concurrencyStart int
+	concurrency      int
 	duration         time.Duration
 	autotune         bool
 	storageClass     string
+	bucketName       string
 }
 
 // Get the max throughput and iops numbers.
@@ -46,7 +50,7 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 
 		throughputHighestGet := uint64(0)
 		throughputHighestPut := uint64(0)
-		var throughputHighestResults []SpeedtestResult
+		var throughputHighestResults []SpeedTestResult
 
 		sendResult := func() {
 			var result madmin.SpeedTestResult
@@ -54,9 +58,9 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 			durationSecs := opts.duration.Seconds()
 
 			result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
-			result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(opts.throughputSize) / uint64(durationSecs)
+			result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(opts.objectSize) / uint64(durationSecs)
 			result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
-			result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(opts.throughputSize) / uint64(durationSecs)
+			result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(opts.objectSize) / uint64(durationSecs)
 			for i := 0; i < len(throughputHighestResults); i++ {
 				errStr := ""
 				if throughputHighestResults[i].Error != "" {
@@ -76,18 +80,18 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 				result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
 					Endpoint:         throughputHighestResults[i].Endpoint,
 					ThroughputPerSec: throughputHighestResults[i].Uploads / uint64(durationSecs),
-					ObjectsPerSec:    throughputHighestResults[i].Uploads / uint64(opts.throughputSize) / uint64(durationSecs),
+					ObjectsPerSec:    throughputHighestResults[i].Uploads / uint64(opts.objectSize) / uint64(durationSecs),
 					Err:              errStr,
 				})
 				result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
 					Endpoint:         throughputHighestResults[i].Endpoint,
 					ThroughputPerSec: throughputHighestResults[i].Downloads / uint64(durationSecs),
-					ObjectsPerSec:    throughputHighestResults[i].Downloads / uint64(opts.throughputSize) / uint64(durationSecs),
+					ObjectsPerSec:    throughputHighestResults[i].Downloads / uint64(opts.objectSize) / uint64(durationSecs),
 					Err:              errStr,
 				})
 			}
 
-			result.Size = opts.throughputSize
+			result.Size = opts.objectSize
 			result.Disks = globalEndpoints.NEndpoints()
 			result.Servers = len(globalNotificationSys.peerClients) + 1
 			result.Version = Version
@@ -104,9 +108,15 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 			default:
 			}
 
-			results := globalNotificationSys.Speedtest(ctx,
-				opts.throughputSize, concurrency,
-				opts.duration, opts.storageClass)
+			sopts := speedTestOpts{
+				objectSize:   opts.objectSize,
+				concurrency:  concurrency,
+				duration:     opts.duration,
+				storageClass: opts.storageClass,
+				bucketName:   opts.bucketName,
+			}
+
+			results := globalNotificationSys.SpeedTest(ctx, sopts)
 			sort.Slice(results, func(i, j int) bool {
 				return results[i].Endpoint < results[j].Endpoint
 			})

cmd/storage-datatypes.go

@@ -21,6 +21,13 @@ import (
 	"time"
 )
 
+// DeleteOptions represents the disk level delete options available for the APIs
+//msgp:ignore DeleteOptions
+type DeleteOptions struct {
+	Recursive bool
+	Force     bool
+}
+
 //go:generate msgp -file=$GOFILE
 
 // DiskInfo is an extended type which returns current

cmd/storage-interface.go

@@ -95,7 +95,7 @@ type StorageAPI interface {
 	ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error)
 	RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
 	CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
-	Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
+	Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error)
 	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
 	StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
 
@@ -223,7 +223,7 @@ func (p *unrecognizedDisk) CheckParts(ctx context.Context, volume string, path s
 	return errDiskNotFound
 }
 
-func (p *unrecognizedDisk) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
+func (p *unrecognizedDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	return errDiskNotFound
 }
 

cmd/storage-rest-client.go

@@ -581,11 +581,12 @@ func (client *storageRESTClient) ListDir(ctx context.Context, volume, dirPath st
 }
 
 // DeleteFile - deletes a file.
-func (client *storageRESTClient) Delete(ctx context.Context, volume string, path string, recursive bool) error {
+func (client *storageRESTClient) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
-	values.Set(storageRESTRecursive, strconv.FormatBool(recursive))
+	values.Set(storageRESTRecursive, strconv.FormatBool(deleteOpts.Recursive))
+	values.Set(storageRESTForceDelete, strconv.FormatBool(deleteOpts.Force))
 
 	respBody, err := client.call(ctx, storageRESTMethodDeleteFile, values, nil, -1)
 	defer xhttp.DrainBody(respBody)

cmd/storage-rest-server.go

@@ -652,8 +652,15 @@ func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Req
 		s.writeErrorResponse(w, err)
 		return
 	}
-
-	err = s.storage.Delete(r.Context(), volume, filePath, recursive)
+	force, err := strconv.ParseBool(r.Form.Get(storageRESTForceDelete))
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	err = s.storage.Delete(r.Context(), volume, filePath, DeleteOptions{
+		Recursive: recursive,
+		Force:     force,
+	})
 	if err != nil {
 		s.writeErrorResponse(w, err)
 	}

cmd/storage-rest_test.go

@@ -383,7 +383,10 @@ func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) {
 	}
 
 	for i, testCase := range testCases {
-		err := storage.Delete(context.Background(), testCase.volumeName, testCase.objectName, false)
+		err := storage.Delete(context.Background(), testCase.volumeName, testCase.objectName, DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 		expectErr := (err != nil)
 
 		if expectErr != testCase.expectErr {

cmd/tier-journal.go

@@ -71,18 +71,9 @@ func initTierDeletionJournal(ctx context.Context) (*tierJournal, error) {
 		tierMemJournal:  newTierMemJoural(1000),
 		tierDiskJournal: newTierDiskJournal(),
 	}
+
 	for _, diskPath := range globalEndpoints.LocalDisksPaths() {
 		j.diskPath = diskPath
-		if err := os.MkdirAll(filepath.Dir(j.JournalPath()), os.FileMode(0o700)); err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-
-		err := j.Open()
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
 
 		go j.deletePending(ctx)  // for existing journal entries from previous MinIO versions
 		go j.processEntries(ctx) // for newer journal entries circa free-versions
@@ -100,10 +91,8 @@ func (jd *tierDiskJournal) rotate() error {
 	if _, err := os.Stat(jd.ReadOnlyPath()); err == nil {
 		return nil
 	}
-	// Close the active journal if present.
-	jd.Close()
-	// Open a new active journal for subsequent journalling.
-	return jd.Open()
+	// Close the active journal if present and delete it.
+	return jd.Close()
 }
 
 type walkFn func(ctx context.Context, objName, rvID, tierName string) error
@@ -117,8 +106,7 @@ func (jd *tierDiskJournal) JournalPath() string {
 }
 
 func (jd *tierDiskJournal) WalkEntries(ctx context.Context, fn walkFn) {
-	err := jd.rotate()
-	if err != nil {
+	if err := jd.rotate(); err != nil {
 		logger.LogIf(ctx, fmt.Errorf("tier-journal: failed to rotate pending deletes journal %s", err))
 		return
 	}
@@ -226,21 +214,17 @@ func (jd *tierDiskJournal) Close() error {
 	if fi, err = f.Stat(); err != nil {
 		return err
 	}
-	defer f.Close()
+	f.Close() // close before rename()
+
 	// Skip renaming active journal if empty.
 	if fi.Size() == tierJournalHdrLen {
-		return nil
+		return os.Remove(jd.JournalPath())
 	}
 
 	jPath := jd.JournalPath()
 	jroPath := jd.ReadOnlyPath()
 	// Rotate active journal to perform pending deletes.
-	err = os.Rename(jPath, jroPath)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return os.Rename(jPath, jroPath)
 }
 
 // Open opens a new active journal. Note: calling Open on an opened journal is a

cmd/update.go

@@ -76,7 +76,7 @@ func releaseTimeToReleaseTag(releaseTime time.Time) string {
 // releaseTagToReleaseTime - reverse of `releaseTimeToReleaseTag()`
 func releaseTagToReleaseTime(releaseTag string) (releaseTime time.Time, err error) {
 	fields := strings.Split(releaseTag, ".")
-	if len(fields) < 2 || len(fields) > 3 {
+	if len(fields) < 2 || len(fields) > 4 {
 		return releaseTime, fmt.Errorf("%s is not a valid release tag", releaseTag)
 	}
 	if fields[0] != "RELEASE" {

cmd/update_test.go

@@ -72,6 +72,14 @@ func TestReleaseTagToNFromTimeConversion(t *testing.T) {
 			time.Now().UTC(), "DEVELOPMENT.GOGET",
 			"DEVELOPMENT.GOGET is not a valid release tag",
 		},
+		{
+			time.Date(2017, time.August, 5, 0, 0, 53, 0, utcLoc),
+			"RELEASE.2017-08-05T00-00-53Z.hotfix", "",
+		},
+		{
+			time.Date(2017, time.August, 5, 0, 0, 53, 0, utcLoc),
+			"RELEASE.2017-08-05T00-00-53Z.hotfix.aaaa", "",
+		},
 	}
 	for i, testCase := range testCases {
 		if testCase.errStr != "" {

cmd/utils.go

@@ -931,6 +931,10 @@ type timedValue struct {
 	// Should be set before calling Get().
 	TTL time.Duration
 
+	// When set to true, return the last cached value
+	// even if updating the value errors out
+	Relax bool
+
 	// Once can be used to initialize values for lazy initialization.
 	// Should be set before calling Get().
 	Once sync.Once
@@ -951,13 +955,16 @@ func (t *timedValue) Get() (interface{}, error) {
 
 	v, err := t.Update()
 	if err != nil {
-		// if update fails, return current
-		// cached value along with error.
-		//
-		// Let the caller decide if they want
-		// to use the returned value based
-		// on error.
-		v = t.get(0)
+		if t.Relax {
+			// if update fails, return current
+			// cached value along with error.
+			//
+			// Let the caller decide if they want
+			// to use the returned value based
+			// on error.
+			v = t.get(0)
+			return v, err
+		}
 		return v, err
 	}
 
@@ -1029,7 +1036,7 @@ func totalNodeCount() uint64 {
 
 // AuditLogOptions takes options for audit logging subsystem activity
 type AuditLogOptions struct {
-	Trigger   string
+	Event     string
 	APIName   string
 	Status    string
 	VersionID string
@@ -1039,7 +1046,8 @@ type AuditLogOptions struct {
 // sends audit logs for internal subsystem activity
 func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogOptions) {
 	entry := audit.NewEntry(globalDeploymentID)
-	entry.Trigger = opts.Trigger
+	entry.Trigger = opts.Event
+	entry.Event = opts.Event
 	entry.Error = opts.Error
 	entry.API.Name = opts.APIName
 	entry.API.Bucket = bucket

cmd/xl-storage-disk-id-check.go

@@ -382,14 +382,14 @@ func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, pa
 	return p.storage.CheckParts(ctx, volume, path, fi)
 }
 
-func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
+func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDelete, volume, path)
 	if err != nil {
 		return err
 	}
 	defer done(&err)
 
-	return p.storage.Delete(ctx, volume, path, recursive)
+	return p.storage.Delete(ctx, volume, path, deleteOpts)
 }
 
 // DeleteVersions deletes slice of versions, it can be same object
@@ -768,7 +768,10 @@ func (p *xlStorageDiskIDCheck) monitorDiskStatus() {
 		if err != nil || len(b) != 10001 {
 			continue
 		}
-		err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, false)
+		err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 		if err == nil {
 			logger.Info("Able to read+write, bringing disk %s online.", p.storage.String())
 			atomic.StoreInt32(&p.health.status, diskHealthOK)

cmd/xl-storage-format-v1.go

@@ -132,6 +132,7 @@ type ObjectPartInfo struct {
 	Number     int    `json:"number"`
 	Size       int64  `json:"size"`
 	ActualSize int64  `json:"actualSize"`
+	Index      []byte `json:"index,omitempty" msg:"index,omitempty"`
 }
 
 // ChecksumInfo - carries checksums of individual scattered parts per disk.

cmd/xl-storage-format-v1_gen.go

@@ -593,6 +593,12 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ActualSize")
 				return
 			}
+		case "index":
+			z.Index, err = dc.ReadBytes(z.Index)
+			if err != nil {
+				err = msgp.WrapError(err, "Index")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -606,9 +612,23 @@ func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 4
+	// omitempty: check for empty values
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
+	if z.Index == nil {
+		zb0001Len--
+		zb0001Mask |= 0x10
+	}
+	// variable map header, size zb0001Len
+	err = en.Append(0x80 | uint8(zb0001Len))
+	if err != nil {
+		return
+	}
+	if zb0001Len == 0 {
+		return
+	}
 	// write "ETag"
-	err = en.Append(0x84, 0xa4, 0x45, 0x54, 0x61, 0x67)
+	err = en.Append(0xa4, 0x45, 0x54, 0x61, 0x67)
 	if err != nil {
 		return
 	}
@@ -647,15 +667,38 @@ func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ActualSize")
 		return
 	}
+	if (zb0001Mask & 0x10) == 0 { // if not empty
+		// write "index"
+		err = en.Append(0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78)
+		if err != nil {
+			return
+		}
+		err = en.WriteBytes(z.Index)
+		if err != nil {
+			err = msgp.WrapError(err, "Index")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
+	// omitempty: check for empty values
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
+	if z.Index == nil {
+		zb0001Len--
+		zb0001Mask |= 0x10
+	}
+	// variable map header, size zb0001Len
+	o = append(o, 0x80|uint8(zb0001Len))
+	if zb0001Len == 0 {
+		return
+	}
 	// string "ETag"
-	o = append(o, 0x84, 0xa4, 0x45, 0x54, 0x61, 0x67)
+	o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67)
 	o = msgp.AppendString(o, z.ETag)
 	// string "Number"
 	o = append(o, 0xa6, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72)
@@ -666,6 +709,11 @@ func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ActualSize"
 	o = append(o, 0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65)
 	o = msgp.AppendInt64(o, z.ActualSize)
+	if (zb0001Mask & 0x10) == 0 { // if not empty
+		// string "index"
+		o = append(o, 0xa5, 0x69, 0x6e, 0x64, 0x65, 0x78)
+		o = msgp.AppendBytes(o, z.Index)
+	}
 	return
 }
 
@@ -711,6 +759,12 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ActualSize")
 				return
 			}
+		case "index":
+			z.Index, bts, err = msgp.ReadBytesBytes(bts, z.Index)
+			if err != nil {
+				err = msgp.WrapError(err, "Index")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -725,7 +779,7 @@ func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *ObjectPartInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.ETag) + 7 + msgp.IntSize + 5 + msgp.Int64Size + 11 + msgp.Int64Size
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.ETag) + 7 + msgp.IntSize + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 6 + msgp.BytesPrefixSize + len(z.Index)
 	return
 }
 

cmd/xl-storage-format-v2.go

@@ -167,6 +167,7 @@ type xlMetaV2Object struct {
 	PartETags          []string          `json:"PartETags" msg:"PartETags,allownil"`             // Part ETags
 	PartSizes          []int64           `json:"PartSizes" msg:"PartSizes"`                      // Part Sizes
 	PartActualSizes    []int64           `json:"PartASizes,omitempty" msg:"PartASizes,allownil"` // Part ActualSizes (compression)
+	PartIndices        [][]byte          `json:"PartIndices,omitempty" msg:"PartIdx,omitempty"`  // Part Indexes (compression)
 	Size               int64             `json:"Size" msg:"Size"`                                // Object version size
 	ModTime            int64             `json:"MTime" msg:"MTime"`                              // Object version modified time
 	MetaSys            map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,allownil"`       // Object version internal metadata
@@ -574,6 +575,9 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 			fi.Parts[i].ETag = j.PartETags[i]
 		}
 		fi.Parts[i].ActualSize = j.PartActualSizes[i]
+		if len(j.PartIndices) > 0 {
+			fi.Parts[i].Index = j.PartIndices[i]
+		}
 	}
 	fi.Erasure.Checksums = make([]ChecksumInfo, len(j.PartSizes))
 	for i := range fi.Parts {
@@ -1471,6 +1475,13 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
 				break
 			}
 		}
+		for i := range fi.Parts {
+			// Only add indices if any.
+			if len(fi.Parts[i].Index) > 0 {
+				ventry.ObjectV2.PartIndices = make([][]byte, len(fi.Parts))
+				break
+			}
+		}
 		for i := range fi.Erasure.Distribution {
 			ventry.ObjectV2.ErasureDist[i] = uint8(fi.Erasure.Distribution[i])
 		}
@@ -1482,6 +1493,9 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
 			}
 			ventry.ObjectV2.PartNumbers[i] = fi.Parts[i].Number
 			ventry.ObjectV2.PartActualSizes[i] = fi.Parts[i].ActualSize
+			if len(ventry.ObjectV2.PartIndices) > 0 {
+				ventry.ObjectV2.PartIndices[i] = fi.Parts[i].Index
+			}
 		}
 
 		tierFVIDKey := ReservedMetadataPrefixLower + tierFVID

cmd/xl-storage-format-v2_gen.go

@@ -935,6 +935,25 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "PartIdx":
+			var zb0009 uint32
+			zb0009, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "PartIndices")
+				return
+			}
+			if cap(z.PartIndices) >= int(zb0009) {
+				z.PartIndices = (z.PartIndices)[:zb0009]
+			} else {
+				z.PartIndices = make([][]byte, zb0009)
+			}
+			for za0008 := range z.PartIndices {
+				z.PartIndices[za0008], err = dc.ReadBytes(z.PartIndices[za0008])
+				if err != nil {
+					err = msgp.WrapError(err, "PartIndices", za0008)
+					return
+				}
+			}
 		case "Size":
 			z.Size, err = dc.ReadInt64()
 			if err != nil {
@@ -956,34 +975,34 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.MetaSys = nil
 			} else {
-				var zb0009 uint32
-				zb0009, err = dc.ReadMapHeader()
+				var zb0010 uint32
+				zb0010, err = dc.ReadMapHeader()
 				if err != nil {
 					err = msgp.WrapError(err, "MetaSys")
 					return
 				}
 				if z.MetaSys == nil {
-					z.MetaSys = make(map[string][]byte, zb0009)
+					z.MetaSys = make(map[string][]byte, zb0010)
 				} else if len(z.MetaSys) > 0 {
 					for key := range z.MetaSys {
 						delete(z.MetaSys, key)
 					}
 				}
-				for zb0009 > 0 {
-					zb0009--
-					var za0008 string
-					var za0009 []byte
-					za0008, err = dc.ReadString()
+				for zb0010 > 0 {
+					zb0010--
+					var za0009 string
+					var za0010 []byte
+					za0009, err = dc.ReadString()
 					if err != nil {
 						err = msgp.WrapError(err, "MetaSys")
 						return
 					}
-					za0009, err = dc.ReadBytes(za0009)
+					za0010, err = dc.ReadBytes(za0010)
 					if err != nil {
-						err = msgp.WrapError(err, "MetaSys", za0008)
+						err = msgp.WrapError(err, "MetaSys", za0009)
 						return
 					}
-					z.MetaSys[za0008] = za0009
+					z.MetaSys[za0009] = za0010
 				}
 			}
 		case "MetaUsr":
@@ -995,34 +1014,34 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.MetaUser = nil
 			} else {
-				var zb0010 uint32
-				zb0010, err = dc.ReadMapHeader()
+				var zb0011 uint32
+				zb0011, err = dc.ReadMapHeader()
 				if err != nil {
 					err = msgp.WrapError(err, "MetaUser")
 					return
 				}
 				if z.MetaUser == nil {
-					z.MetaUser = make(map[string]string, zb0010)
+					z.MetaUser = make(map[string]string, zb0011)
 				} else if len(z.MetaUser) > 0 {
 					for key := range z.MetaUser {
 						delete(z.MetaUser, key)
 					}
 				}
-				for zb0010 > 0 {
-					zb0010--
-					var za0010 string
+				for zb0011 > 0 {
+					zb0011--
 					var za0011 string
-					za0010, err = dc.ReadString()
+					var za0012 string
+					za0011, err = dc.ReadString()
 					if err != nil {
 						err = msgp.WrapError(err, "MetaUser")
 						return
 					}
-					za0011, err = dc.ReadString()
+					za0012, err = dc.ReadString()
 					if err != nil {
-						err = msgp.WrapError(err, "MetaUser", za0010)
+						err = msgp.WrapError(err, "MetaUser", za0011)
 						return
 					}
-					z.MetaUser[za0010] = za0011
+					z.MetaUser[za0011] = za0012
 				}
 			}
 		default:
@@ -1038,9 +1057,23 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 17
+	// omitempty: check for empty values
+	zb0001Len := uint32(18)
+	var zb0001Mask uint32 /* 18 bits */
+	if z.PartIndices == nil {
+		zb0001Len--
+		zb0001Mask |= 0x2000
+	}
+	// variable map header, size zb0001Len
+	err = en.WriteMapHeader(zb0001Len)
+	if err != nil {
+		return
+	}
+	if zb0001Len == 0 {
+		return
+	}
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x11, 0xa2, 0x49, 0x44)
+	err = en.Append(0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -1218,6 +1251,25 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
+	if (zb0001Mask & 0x2000) == 0 { // if not empty
+		// write "PartIdx"
+		err = en.Append(0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78)
+		if err != nil {
+			return
+		}
+		err = en.WriteArrayHeader(uint32(len(z.PartIndices)))
+		if err != nil {
+			err = msgp.WrapError(err, "PartIndices")
+			return
+		}
+		for za0008 := range z.PartIndices {
+			err = en.WriteBytes(z.PartIndices[za0008])
+			if err != nil {
+				err = msgp.WrapError(err, "PartIndices", za0008)
+				return
+			}
+		}
+	}
 	// write "Size"
 	err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
 	if err != nil {
@@ -1254,15 +1306,15 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "MetaSys")
 			return
 		}
-		for za0008, za0009 := range z.MetaSys {
-			err = en.WriteString(za0008)
+		for za0009, za0010 := range z.MetaSys {
+			err = en.WriteString(za0009)
 			if err != nil {
 				err = msgp.WrapError(err, "MetaSys")
 				return
 			}
-			err = en.WriteBytes(za0009)
+			err = en.WriteBytes(za0010)
 			if err != nil {
-				err = msgp.WrapError(err, "MetaSys", za0008)
+				err = msgp.WrapError(err, "MetaSys", za0009)
 				return
 			}
 		}
@@ -1283,15 +1335,15 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "MetaUser")
 			return
 		}
-		for za0010, za0011 := range z.MetaUser {
-			err = en.WriteString(za0010)
+		for za0011, za0012 := range z.MetaUser {
+			err = en.WriteString(za0011)
 			if err != nil {
 				err = msgp.WrapError(err, "MetaUser")
 				return
 			}
-			err = en.WriteString(za0011)
+			err = en.WriteString(za0012)
 			if err != nil {
-				err = msgp.WrapError(err, "MetaUser", za0010)
+				err = msgp.WrapError(err, "MetaUser", za0011)
 				return
 			}
 		}
@@ -1302,9 +1354,20 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 17
+	// omitempty: check for empty values
+	zb0001Len := uint32(18)
+	var zb0001Mask uint32 /* 18 bits */
+	if z.PartIndices == nil {
+		zb0001Len--
+		zb0001Mask |= 0x2000
+	}
+	// variable map header, size zb0001Len
+	o = msgp.AppendMapHeader(o, zb0001Len)
+	if zb0001Len == 0 {
+		return
+	}
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44)
+	o = append(o, 0xa2, 0x49, 0x44)
 	o = msgp.AppendBytes(o, (z.VersionID)[:])
 	// string "DDir"
 	o = append(o, 0xa4, 0x44, 0x44, 0x69, 0x72)
@@ -1365,6 +1428,14 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 			o = msgp.AppendInt64(o, z.PartActualSizes[za0007])
 		}
 	}
+	if (zb0001Mask & 0x2000) == 0 { // if not empty
+		// string "PartIdx"
+		o = append(o, 0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78)
+		o = msgp.AppendArrayHeader(o, uint32(len(z.PartIndices)))
+		for za0008 := range z.PartIndices {
+			o = msgp.AppendBytes(o, z.PartIndices[za0008])
+		}
+	}
 	// string "Size"
 	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
 	o = msgp.AppendInt64(o, z.Size)
@@ -1377,9 +1448,9 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 		o = msgp.AppendNil(o)
 	} else {
 		o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys)))
-		for za0008, za0009 := range z.MetaSys {
-			o = msgp.AppendString(o, za0008)
-			o = msgp.AppendBytes(o, za0009)
+		for za0009, za0010 := range z.MetaSys {
+			o = msgp.AppendString(o, za0009)
+			o = msgp.AppendBytes(o, za0010)
 		}
 	}
 	// string "MetaUsr"
@@ -1388,9 +1459,9 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 		o = msgp.AppendNil(o)
 	} else {
 		o = msgp.AppendMapHeader(o, uint32(len(z.MetaUser)))
-		for za0010, za0011 := range z.MetaUser {
-			o = msgp.AppendString(o, za0010)
+		for za0011, za0012 := range z.MetaUser {
 			o = msgp.AppendString(o, za0011)
+			o = msgp.AppendString(o, za0012)
 		}
 	}
 	return
@@ -1575,6 +1646,25 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					}
 				}
 			}
+		case "PartIdx":
+			var zb0009 uint32
+			zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PartIndices")
+				return
+			}
+			if cap(z.PartIndices) >= int(zb0009) {
+				z.PartIndices = (z.PartIndices)[:zb0009]
+			} else {
+				z.PartIndices = make([][]byte, zb0009)
+			}
+			for za0008 := range z.PartIndices {
+				z.PartIndices[za0008], bts, err = msgp.ReadBytesBytes(bts, z.PartIndices[za0008])
+				if err != nil {
+					err = msgp.WrapError(err, "PartIndices", za0008)
+					return
+				}
+			}
 		case "Size":
 			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
@@ -1592,34 +1682,34 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				bts = bts[1:]
 				z.MetaSys = nil
 			} else {
-				var zb0009 uint32
-				zb0009, bts, err = msgp.ReadMapHeaderBytes(bts)
+				var zb0010 uint32
+				zb0010, bts, err = msgp.ReadMapHeaderBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "MetaSys")
 					return
 				}
 				if z.MetaSys == nil {
-					z.MetaSys = make(map[string][]byte, zb0009)
+					z.MetaSys = make(map[string][]byte, zb0010)
 				} else if len(z.MetaSys) > 0 {
 					for key := range z.MetaSys {
 						delete(z.MetaSys, key)
 					}
 				}
-				for zb0009 > 0 {
-					var za0008 string
-					var za0009 []byte
-					zb0009--
-					za0008, bts, err = msgp.ReadStringBytes(bts)
+				for zb0010 > 0 {
+					var za0009 string
+					var za0010 []byte
+					zb0010--
+					za0009, bts, err = msgp.ReadStringBytes(bts)
 					if err != nil {
 						err = msgp.WrapError(err, "MetaSys")
 						return
 					}
-					za0009, bts, err = msgp.ReadBytesBytes(bts, za0009)
+					za0010, bts, err = msgp.ReadBytesBytes(bts, za0010)
 					if err != nil {
-						err = msgp.WrapError(err, "MetaSys", za0008)
+						err = msgp.WrapError(err, "MetaSys", za0009)
 						return
 					}
-					z.MetaSys[za0008] = za0009
+					z.MetaSys[za0009] = za0010
 				}
 			}
 		case "MetaUsr":
@@ -1627,34 +1717,34 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				bts = bts[1:]
 				z.MetaUser = nil
 			} else {
-				var zb0010 uint32
-				zb0010, bts, err = msgp.ReadMapHeaderBytes(bts)
+				var zb0011 uint32
+				zb0011, bts, err = msgp.ReadMapHeaderBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "MetaUser")
 					return
 				}
 				if z.MetaUser == nil {
-					z.MetaUser = make(map[string]string, zb0010)
+					z.MetaUser = make(map[string]string, zb0011)
 				} else if len(z.MetaUser) > 0 {
 					for key := range z.MetaUser {
 						delete(z.MetaUser, key)
 					}
 				}
-				for zb0010 > 0 {
-					var za0010 string
+				for zb0011 > 0 {
 					var za0011 string
-					zb0010--
-					za0010, bts, err = msgp.ReadStringBytes(bts)
+					var za0012 string
+					zb0011--
+					za0011, bts, err = msgp.ReadStringBytes(bts)
 					if err != nil {
 						err = msgp.WrapError(err, "MetaUser")
 						return
 					}
-					za0011, bts, err = msgp.ReadStringBytes(bts)
+					za0012, bts, err = msgp.ReadStringBytes(bts)
 					if err != nil {
-						err = msgp.WrapError(err, "MetaUser", za0010)
+						err = msgp.WrapError(err, "MetaUser", za0011)
 						return
 					}
-					z.MetaUser[za0010] = za0011
+					z.MetaUser[za0011] = za0012
 				}
 			}
 		default:
@@ -1675,18 +1765,22 @@ func (z *xlMetaV2Object) Msgsize() (s int) {
 	for za0005 := range z.PartETags {
 		s += msgp.StringPrefixSize + len(z.PartETags[za0005])
 	}
-	s += 10 + msgp.ArrayHeaderSize + (len(z.PartSizes) * (msgp.Int64Size)) + 11 + msgp.ArrayHeaderSize + (len(z.PartActualSizes) * (msgp.Int64Size)) + 5 + msgp.Int64Size + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize
+	s += 10 + msgp.ArrayHeaderSize + (len(z.PartSizes) * (msgp.Int64Size)) + 11 + msgp.ArrayHeaderSize + (len(z.PartActualSizes) * (msgp.Int64Size)) + 8 + msgp.ArrayHeaderSize
+	for za0008 := range z.PartIndices {
+		s += msgp.BytesPrefixSize + len(z.PartIndices[za0008])
+	}
+	s += 5 + msgp.Int64Size + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize
 	if z.MetaSys != nil {
-		for za0008, za0009 := range z.MetaSys {
-			_ = za0009
-			s += msgp.StringPrefixSize + len(za0008) + msgp.BytesPrefixSize + len(za0009)
+		for za0009, za0010 := range z.MetaSys {
+			_ = za0010
+			s += msgp.StringPrefixSize + len(za0009) + msgp.BytesPrefixSize + len(za0010)
 		}
 	}
 	s += 8 + msgp.MapHeaderSize
 	if z.MetaUser != nil {
-		for za0010, za0011 := range z.MetaUser {
-			_ = za0011
-			s += msgp.StringPrefixSize + len(za0010) + msgp.StringPrefixSize + len(za0011)
+		for za0011, za0012 := range z.MetaUser {
+			_ = za0012
+			s += msgp.StringPrefixSize + len(za0011) + msgp.StringPrefixSize + len(za0012)
 		}
 	}
 	return

cmd/xl-storage.go

@@ -818,7 +818,7 @@ func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bo
 	}
 
 	if forceDelete {
-		err = s.moveToTrash(volumeDir, true)
+		err = s.moveToTrash(volumeDir, true, true)
 	} else {
 		err = Remove(volumeDir)
 	}
@@ -903,7 +903,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	if !isXL2V1Format(buf) {
 		// Delete the meta file, if there are no more versions the
 		// top level parent is automatically removed.
-		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true)
+		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)
 	}
 
 	var xlMeta xlMetaV2
@@ -939,7 +939,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 			if err = checkPathLength(filePath); err != nil {
 				return err
 			}
-			if err = s.moveToTrash(filePath, true); err != nil {
+			if err = s.moveToTrash(filePath, true, false); err != nil {
 				if err != errFileNotFound {
 					return err
 				}
@@ -959,9 +959,9 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	}
 
 	// Move xl.meta to trash
-	err = s.moveToTrash(pathJoin(volumeDir, path, xlStorageFormatFile), false)
+	err = s.moveToTrash(pathJoin(volumeDir, path, xlStorageFormatFile), false, false)
 	if err == nil || err == errFileNotFound {
-		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false)
+		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false, false)
 	}
 	return err
 }
@@ -985,19 +985,36 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions
 	return errs
 }
 
-func (s *xlStorage) moveToTrash(filePath string, recursive bool) error {
+func (s *xlStorage) moveToTrash(filePath string, recursive, force bool) error {
 	pathUUID := mustGetUUID()
+	targetPath := pathutil.Join(s.diskPath, minioMetaTmpDeletedBucket, pathUUID)
+
+	var renameFn func(source, target string) error
 	if recursive {
-		return renameAll(filePath, pathutil.Join(s.diskPath, minioMetaTmpDeletedBucket, pathUUID))
+		renameFn = renameAll
+	} else {
+		renameFn = Rename
 	}
-	return Rename(filePath, pathutil.Join(s.diskPath, minioMetaTmpDeletedBucket, pathUUID))
+
+	if err := renameFn(filePath, targetPath); err != nil {
+		return err
+	}
+	// immediately purge the target
+	if force {
+		removeAll(targetPath)
+	}
+
+	return nil
 }
 
 // DeleteVersion - deletes FileInfo metadata for path at `xl.meta`. forceDelMarker
 // will force creating a new `xl.meta` to create a new delete marker
 func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) error {
 	if HasSuffix(path, SlashSeparator) {
-		return s.Delete(ctx, volume, path, false)
+		return s.Delete(ctx, volume, path, DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 	}
 
 	buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
@@ -1035,7 +1052,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 	if !isXL2V1Format(buf) {
 		// Delete the meta file, if there are no more versions the
 		// top level parent is automatically removed.
-		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true)
+		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)
 	}
 
 	var xlMeta xlMetaV2
@@ -1065,7 +1082,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		if err = checkPathLength(filePath); err != nil {
 			return err
 		}
-		if err = s.moveToTrash(filePath, true); err != nil {
+		if err = s.moveToTrash(filePath, true, false); err != nil {
 			if err != errFileNotFound {
 				return err
 			}
@@ -1088,9 +1105,9 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		return err
 	}
 
-	err = s.moveToTrash(filePath, false)
+	err = s.moveToTrash(filePath, false, false)
 	if err == nil || err == errFileNotFound {
-		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false)
+		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false, false)
 	}
 	return err
 }
@@ -2014,7 +2031,7 @@ func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string,
 // move up the tree, deleting empty parent directories until it finds one
 // with files in it. Returns nil for a non-empty directory even when
 // recursive is set to false.
-func (s *xlStorage) deleteFile(basePath, deletePath string, recursive bool) error {
+func (s *xlStorage) deleteFile(basePath, deletePath string, recursive, force bool) error {
 	if basePath == "" || deletePath == "" {
 		return nil
 	}
@@ -2027,7 +2044,7 @@ func (s *xlStorage) deleteFile(basePath, deletePath string, recursive bool) erro
 
 	var err error
 	if recursive {
-		err = s.moveToTrash(deletePath, true)
+		err = s.moveToTrash(deletePath, true, force)
 	} else {
 		err = Remove(deletePath)
 	}
@@ -2060,13 +2077,13 @@ func (s *xlStorage) deleteFile(basePath, deletePath string, recursive bool) erro
 
 	// Delete parent directory obviously not recursively. Errors for
 	// parent directories shouldn't trickle down.
-	s.deleteFile(basePath, deletePath, false)
+	s.deleteFile(basePath, deletePath, false, false)
 
 	return nil
 }
 
 // DeleteFile - delete a file at path.
-func (s *xlStorage) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
+func (s *xlStorage) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
 		return err
@@ -2092,7 +2109,7 @@ func (s *xlStorage) Delete(ctx context.Context, volume string, path string, recu
 	}
 
 	// Delete file and delete parent directory as well if it's empty.
-	return s.deleteFile(volumeDir, filePath, recursive)
+	return s.deleteFile(volumeDir, filePath, deleteOpts.Recursive, deleteOpts.Force)
 }
 
 func skipAccessChecks(volume string) (ok bool) {
@@ -2270,7 +2287,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		// legacy data dir means its old content, honor system umask.
 		if err = mkdirAll(legacyDataPath, 0o777); err != nil {
 			// any failed mkdir-calls delete them.
-			s.deleteFile(dstVolumeDir, legacyDataPath, true)
+			s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 			return osErrToFileErr(err)
 		}
 
@@ -2282,7 +2299,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 
 			if err = Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil {
 				// Any failed rename calls un-roll previous transaction.
-				s.deleteFile(dstVolumeDir, legacyDataPath, true)
+				s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 
 				return osErrToFileErr(err)
 			}
@@ -2327,7 +2344,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 	if err = xlMeta.AddVersion(fi); err != nil {
 		if legacyPreserved {
 			// Any failed rename calls un-roll previous transaction.
-			s.deleteFile(dstVolumeDir, legacyDataPath, true)
+			s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 		}
 		return err
 	}
@@ -2338,7 +2355,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		logger.LogIf(ctx, err)
 		if legacyPreserved {
 			// Any failed rename calls un-roll previous transaction.
-			s.deleteFile(dstVolumeDir, legacyDataPath, true)
+			s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 		}
 		return errFileCorrupt
 	}
@@ -2347,7 +2364,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), dstBuf); err != nil {
 			if legacyPreserved {
 				// Any failed rename calls un-roll previous transaction.
-				s.deleteFile(dstVolumeDir, legacyDataPath, true)
+				s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 			}
 			return osErrToFileErr(err)
 		}
@@ -2355,19 +2372,19 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 
 		// renameAll only for objects that have xl.meta not saved inline.
 		if len(fi.Data) == 0 && fi.Size > 0 {
-			s.moveToTrash(dstDataPath, true)
+			s.moveToTrash(dstDataPath, true, false)
 			if healing {
 				// If we are healing we should purge any legacyDataPath content,
 				// that was previously preserved during PutObject() call
 				// on a versioned bucket.
-				s.moveToTrash(legacyDataPath, true)
+				s.moveToTrash(legacyDataPath, true, false)
 			}
 			if err = renameAll(srcDataPath, dstDataPath); err != nil {
 				if legacyPreserved {
 					// Any failed rename calls un-roll previous transaction.
-					s.deleteFile(dstVolumeDir, legacyDataPath, true)
+					s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 				}
-				s.deleteFile(dstVolumeDir, dstDataPath, false)
+				s.deleteFile(dstVolumeDir, dstDataPath, false, false)
 				return osErrToFileErr(err)
 			}
 		}
@@ -2376,9 +2393,9 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		if err = renameAll(srcFilePath, dstFilePath); err != nil {
 			if legacyPreserved {
 				// Any failed rename calls un-roll previous transaction.
-				s.deleteFile(dstVolumeDir, legacyDataPath, true)
+				s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 			}
-			s.deleteFile(dstVolumeDir, dstFilePath, false)
+			s.deleteFile(dstVolumeDir, dstFilePath, false, false)
 			return osErrToFileErr(err)
 		}
 
@@ -2386,16 +2403,16 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		// movement, this is to ensure that previous data references can co-exist for
 		// any recoverability.
 		if oldDstDataPath != "" {
-			s.moveToTrash(oldDstDataPath, true)
+			s.moveToTrash(oldDstDataPath, true, false)
 		}
 	} else {
 		// Write meta-file directly, no data
 		if err = s.WriteAll(ctx, dstVolume, pathJoin(dstPath, xlStorageFormatFile), dstBuf); err != nil {
 			if legacyPreserved {
 				// Any failed rename calls un-roll previous transaction.
-				s.deleteFile(dstVolumeDir, legacyDataPath, true)
+				s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
 			}
-			s.deleteFile(dstVolumeDir, dstFilePath, false)
+			s.deleteFile(dstVolumeDir, dstFilePath, false, false)
 			return err
 		}
 	}
@@ -2484,7 +2501,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 
 	// Remove parent dir of the source file if empty
 	parentDir := pathutil.Dir(srcFilePath)
-	s.deleteFile(srcVolumeDir, parentDir, false)
+	s.deleteFile(srcVolumeDir, parentDir, false, false)
 
 	return nil
 }

cmd/xl-storage_test.go

@@ -933,14 +933,20 @@ func TestXLStorageListDir(t *testing.T) {
 			t.Fatalf("Unable to initialize xlStorage, %s", err)
 		}
 
-		if err = xlStorageNew.Delete(context.Background(), "mybucket", "myobject", false); err != errVolumeAccessDenied {
+		if err = xlStorageNew.Delete(context.Background(), "mybucket", "myobject", DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		}); err != errVolumeAccessDenied {
 			t.Errorf("expected: %s, got: %s", errVolumeAccessDenied, err)
 		}
 	}
 
 	// TestXLStorage for delete on an removed disk.
 	// should fail with disk not found.
-	err = xlStorageDeletedStorage.Delete(context.Background(), "del-vol", "my-file", false)
+	err = xlStorageDeletedStorage.Delete(context.Background(), "del-vol", "my-file", DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != errDiskNotFound {
 		t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
 	}
@@ -1029,7 +1035,10 @@ func TestXLStorageDeleteFile(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		if err = xlStorage.Delete(context.Background(), testCase.srcVol, testCase.srcPath, false); err != testCase.expectedErr {
+		if err = xlStorage.Delete(context.Background(), testCase.srcVol, testCase.srcPath, DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		}); err != testCase.expectedErr {
 			t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
@@ -1054,7 +1063,10 @@ func TestXLStorageDeleteFile(t *testing.T) {
 			t.Fatalf("Unable to initialize xlStorage, %s", err)
 		}
 
-		if err = xlStorageNew.Delete(context.Background(), "mybucket", "myobject", false); err != errVolumeAccessDenied {
+		if err = xlStorageNew.Delete(context.Background(), "mybucket", "myobject", DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		}); err != errVolumeAccessDenied {
 			t.Errorf("expected: %s, got: %s", errVolumeAccessDenied, err)
 		}
 	}
@@ -1072,7 +1084,10 @@ func TestXLStorageDeleteFile(t *testing.T) {
 
 	// TestXLStorage for delete on an removed disk.
 	// should fail with disk not found.
-	err = xlStorageDeletedStorage.Delete(context.Background(), "del-vol", "my-file", false)
+	err = xlStorageDeletedStorage.Delete(context.Background(), "del-vol", "my-file", DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != errDiskNotFound {
 		t.Errorf("Expected: \"Disk not found\", got \"%s\"", err)
 	}
@@ -1907,7 +1922,10 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal("expected to fail bitrot check")
 	}
 
-	if err := storage.Delete(context.Background(), volName, fileName, false); err != nil {
+	if err := storage.Delete(context.Background(), volName, fileName, DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	}); err != nil {
 		t.Fatal(err)
 	}
 

cmd/xl-storage_windows_test.go

@@ -70,7 +70,10 @@ func TestUNCPaths(t *testing.T) {
 			} else if err == nil && !test.pass {
 				t.Error(err)
 			}
-			fs.Delete(context.Background(), "voldir", test.objName, false)
+			fs.Delete(context.Background(), "voldir", test.objName, DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 		})
 	}
 }

docs/bucket/replication/README.md

@@ -244,6 +244,7 @@ Replication from a source bucket to multiple destination buckets is supported. F
 Note that on the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` after replication succeeds to each of the targets. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.
 
 ### Interaction with extended Bucket Versioning configuration
+
 When Bucket Versioning with excluded prefixes are configured objects matching these prefixes are excluded from being versioned and replicated.
 
 ```
@@ -261,6 +262,7 @@ When Bucket Versioning with excluded prefixes are configured objects matching th
         <!-- .. up to 10 prefixes in all -->
 </VersioningConfiguration>
 ```
+
 In the above sample config, objects under prefixes matching any of the `ExcludedPrefixes` glob patterns will neither be versioned nor replicated.
 
 ## Explore Further

docs/bucket/versioning/DESIGN.md

@@ -131,7 +131,7 @@ Inline data is optional. If no inline data is present, it is encoded as 0 bytes.
 | Entry               | Encoding                    | Content
 | --------------------|-----------------------------|----------------------------------------
 | xlMetaInlineDataVer | byte                        | version identifier
-| id -> data          | msgp map[string][]byte      | Map of string id -> byte content
+| id -> data          | msgp `map[string][]byte`      | Map of string id -> byte content
 
 Currently only xlMetaInlineDataVer == 1 exists.
 

docs/bucket/versioning/README.md

@@ -64,12 +64,15 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen
 ```
 
 ## MinIO extension to Bucket Versioning
+
 ### Motivation
-**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on https://subnet.min.io.**
+
+**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on <https://subnet.min.io>.**
 
 Spark/Hadoop workloads which use Hadoop MR Committer v1/v2 algorithm upload objects to a temporary prefix in a bucket. These objects are 'renamed' to a different prefix on Job commit. Object storage admins are forced to configure separate ILM policies to expire these objects and their versions to reclaim space.
 
 ### Solution
+
 To exclude objects under a list of prefix (glob) patterns from being versioned, you can send the following versioning configuration with Status set to `Enabled`.
 
 ```
@@ -92,6 +95,7 @@ To exclude objects under a list of prefix (glob) patterns from being versioned,
 ```
 
 ### Features
+
 - Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if bucket has replication configured.
 - Objects matching these prefixes will also not leave `null` delete markers, dramatically reduces namespace pollution while keeping the benefits of replication.
 - Users with explicit permissions or the root credential can configure the versioning state of any bucket.

docs/compression/README.md

@@ -54,9 +54,9 @@ The compression settings may also be set through environment variables.
 When set, environment variables override the defined `compress` config settings in the server config.
 
 ```bash
-export MINIO_COMPRESS="on"
-export MINIO_COMPRESS_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin"
-export MINIO_COMPRESS_MIME_TYPES="text/*,application/json,application/xml"
+export MINIO_COMPRESSION_ENABLE="on"
+export MINIO_COMPRESSION_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin"
+export MINIO_COMPRESSION_MIME_TYPES="text/*,application/json,application/xml"
 ```
 
 ### 3. Compression + Encryption
@@ -76,7 +76,7 @@ To enable compression+encryption use:
 ~ mc admin config set myminio compression allow_encryption=on
 ```
 
-Or alternatively through the environment variable `MINIO_COMPRESS_ALLOW_ENCRYPTION=on`.
+Or alternatively through the environment variable `MINIO_COMPRESSION_ALLOW_ENCRYPTION=on`.
 
 ### 4. Excluded Types
 

docs/distributed/DECOMMISSION.md

@@ -1,14 +1,14 @@
 # Decommissioning
 
-Decommissiong is a mechanism in MinIO to drain older pools (usually with old hardware) and migrate the content from such pools to a newer pools (usually better hardware). Decommissioning spreads the data across all pools - for example if you decommission `pool1`, all the data from `pool1` shall be spread across `pool2` and `pool3` respectively.
+Decommissiong is a mechanism in MinIO to drain older pools (usually with old hardware) and migrate the content from such pools to a newer pools (usually better hardware). Decommissioning spreads the data across all pools - for example, if you decommission `pool1`, all the data from `pool1` spreads across `pool2` and `pool3`.
 
 ## Features
 
-- A pool in decommission still allows READ access to all its contents, newer WRITEs will be automatically scheduled to only new pools.
-- All versioned buckets maintain the same order for "versions" for each objects after being decommissioned to the newer pools.
-- A pool in decommission resumes from where it was left off (for example - in-case of cluster restarts or restarts attempted after a failed decommission attempt).
+- A pool in decommission still allows READ access to all its contents, newer WRITEs will automatically be scheduled to only pools not in decommission status.
+- All versioned buckets maintain the same order for "versions" for each object after being decommissioned to the other pools.
+- A pool interrupted during the decommission process, such as for a cluster restart, resumes from where it left off.
 
-## How to decommission a pool?
+## How to decommission a pool
 
 ```
 λ mc admin decommission start alias/ http://minio{1...2}/data{1...4}
@@ -49,7 +49,7 @@ Decommission of pool http://minio{1...2}/data{1...4} is complete, you may now re
 ERROR: This pool is not scheduled for decommissioning currently.
 ```
 
-## Canceling a decommission?
+## Canceling a decommission
 
 Stop an on-going decommission in progress, mainly used in situations when the load may be too high and you may want to schedule the decommission at a later point in time.
 
@@ -73,7 +73,7 @@ Stop an on-going decommission in progress, mainly used in situations when the lo
 └─────┴─────────────────────────────────┴──────────────────────────────────┴────────────────────┘
 ```
 
-If for some reason decommission fails in between, the `status` will indicate decommission as failed instead.
+If the decommission process fails for any reason, the status indicates failed.
 
 ```
 λ mc admin decommission status alias/
@@ -84,13 +84,13 @@ If for some reason decommission fails in between, the `status` will indicate dec
 └─────┴─────────────────────────────────┴──────────────────────────────────┴──────────────────┘
 ```
 
-## Restart a canceled or failed decommission?
+## Restart a canceled or failed decommission
 
 ```
 λ mc admin decommission start alias/ http://minio{1...2}/data{1...4}
 ```
 
-## When decommission is 'Complete'?
+## When decommission is 'Complete'
 
 Once decommission is complete, it will be indicated with *Complete* status.  *Complete* means that now you can now safely remove the first pool argument from the MinIO command line.
 
@@ -103,17 +103,17 @@ Once decommission is complete, it will be indicated with *Complete* status.  *Co
 └─────┴─────────────────────────────────┴──────────────────────────────────┴──────────┘
 ```
 
-- On baremetal setups if you have `MINIO_VOLUMES="http://minio{1...2}/data{1...4} http://minio{3...4}/data{1...4}"` you can remove the first argument `http://minio{1...2}/data{1...4}` to update your `MINIO_VOLUMES` setting and using `systemctl restart minio` on all the servers in the setup in parallel.
+- On baremetal setups, if you have `MINIO_VOLUMES="http://minio{1...2}/data{1...4} http://minio{3...4}/data{1...4}"`, you can remove the first argument `http://minio{1...2}/data{1...4}` to update your `MINIO_VOLUMES` setting, then restart all the servers in the setup in parallel using `systemctl restart minio`.
 
-- On Kubernetes setups MinIO the statefulset specification needs to be modified by changing the command line input for the MinIO container. Once the relevant changes are done proceed to execute `kubectl apply -f statefulset.yaml`.
+- On Kubernetes setups, the statefulset specification needs to be modified by changing the command line input for the MinIO container. Once the relevant changes are done, proceed to execute `kubectl apply -f statefulset.yaml`.
 
-- On Operator based MinIO deployments you need to modify the `tenant.yaml` specification and modify the `pools:` section from two entries to a single entry, once relevant changes are done proceed to execute `kubectl apply -f tenant.yaml`.
+- On Operator based MinIO deployments, you need to modify the `tenant.yaml` specification and modify the `pools:` section from two entries to a single entry. After making relevant changes, proceed to execute `kubectl apply -f tenant.yaml`.
 
 > Without a 'Complete' status any 'Active' or 'Draining' pool(s) are not allowed to be removed once configured.
 
 ## NOTE
 
-- Empty delete marker's i.e objects with no other successor versions are not transitioned to the new pool, to avoid any empty metadata being recreated on the newer pool. We do not think this is needed, please open a GitHub issue if you think otherwise.
+- Empty delete markers (such as for objects with no other successor versions) do not transition to the new pool to avoid creating empty metadata on the other pool(s). If you believe transitioning empty delete markers is required, open a GitHub issue.
 
 ## TODO
 

docs/distributed/README.md

@@ -28,7 +28,6 @@ MinIO follows strict **read-after-write** and **list-after-write** consistency m
 
 **If MinIO distributed setup is using NFS volumes underneath it is not guaranteed MinIO will provide these consistency guarantees since NFS is not strictly consistent (If you must use NFS we recommend that you atleast use NFSv4 instead of NFSv3 for relatively better outcomes).**
 
-
 ## Get started
 
 If you're aware of stand-alone MinIO set up, the process remains largely the same. MinIO server automatically switches to stand-alone or distributed mode, depending on the command line parameters.

docs/distributed/decom-compressed-sse-s3.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+    set -x
+fi
+
+pkill minio
+rm -rf /tmp/xl
+
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+	chmod +x mc
+fi
+
+export CI=true
+export MINIO_COMPRESSION_ENABLE="on"
+export MINIO_COMPRESSION_EXTENSIONS=".go"
+export MINIO_COMPRESSION_MIME_TYPES="application/*"
+export MINIO_COMPRESSION_ALLOW_ENCRYPTION="on"
+export MINIO_KMS_AUTO_ENCRYPTION=on
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
+
+(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)&
+pid=$!
+
+sleep 2
+
+./mc admin user add myminio/ minio123 minio123
+./mc admin user add myminio/ minio12345 minio12345
+
+./mc admin policy add myminio/ rw ./docs/distributed/rw.json
+./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+
+./mc admin policy set myminio/ rw user=minio123
+./mc admin policy set myminio/ lake,rw user=minio12345
+
+./mc mb -l myminio/versioned
+
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+## Soft delete (creates delete markers)
+./mc rm -r --force myminio/versioned >/dev/null
+
+## mirror again to create another set of version on top
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum)
+
+user_count=$(./mc admin user list myminio/ | wc -l)
+policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+kill $pid
+
+(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
+pid=$!
+
+sleep 2
+
+expanded_user_count=$(./mc admin user list myminio/ | wc -l)
+expanded_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $expanded_user_count ]; then
+    echo "BUG: original user count differs from expanded setup"
+    exit 1
+fi
+
+if [ $policy_count -ne $expanded_policy_count ]; then
+    echo "BUG: original policy count  differs from expanded setup"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "expected versioning enabled after expansion"
+    exit 1
+fi
+
+./mc mirror cmd myminio/versioned/ --quiet >/dev/null
+
+./mc ls -r myminio/versioned/ > expanded_ns.txt
+./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt
+
+./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+
+until $(./mc admin decom status myminio/ | grep -q Complete)
+do
+    echo "waiting for decom to finish..."
+    sleep 1
+done
+
+kill $pid
+
+(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log)&
+pid=$!
+
+sleep 2
+
+decom_user_count=$(./mc admin user list myminio/ | wc -l)
+decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $decom_user_count ]; then
+    echo "BUG: original user count differs after decommission"
+    exit 1
+fi
+
+if [ $policy_count -ne $decom_policy_count ]; then
+    echo "BUG: original policy count differs after decommission"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected versioning enabled after decommission"
+    exit 1
+fi
+
+got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum)
+if [ "${expected_checksum}" != "${got_checksum}" ]; then
+    echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}"
+    exit 1
+fi
+
+./mc ls -r myminio/versioned > decommissioned_ns.txt
+./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt
+
+out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+# kill $pid

docs/distributed/decom-encrypted-sse-s3.sh

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+    set -x
+fi
+
+pkill minio
+rm -rf /tmp/xl
+
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+	chmod +x mc
+fi
+
+export CI=true
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
+
+(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)&
+pid=$!
+
+sleep 2
+
+./mc admin user add myminio/ minio123 minio123
+./mc admin user add myminio/ minio12345 minio12345
+
+./mc admin policy add myminio/ rw ./docs/distributed/rw.json
+./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+
+./mc admin policy set myminio/ rw user=minio123
+./mc admin policy set myminio/ lake,rw user=minio12345
+
+./mc mb -l myminio/versioned
+
+./mc encrypt set sse-s3 myminio/versioned
+
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+## Soft delete (creates delete markers)
+./mc rm -r --force myminio/versioned >/dev/null
+
+## mirror again to create another set of version on top
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum)
+
+user_count=$(./mc admin user list myminio/ | wc -l)
+policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+kill $pid
+
+(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
+pid=$!
+
+sleep 2
+
+expanded_user_count=$(./mc admin user list myminio/ | wc -l)
+expanded_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $expanded_user_count ]; then
+    echo "BUG: original user count differs from expanded setup"
+    exit 1
+fi
+
+if [ $policy_count -ne $expanded_policy_count ]; then
+    echo "BUG: original policy count  differs from expanded setup"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "expected versioning enabled after expansion"
+    exit 1
+fi
+
+./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "expected encryption enabled after expansion"
+    exit 1
+fi
+
+./mc mirror cmd myminio/versioned/ --quiet >/dev/null
+
+./mc ls -r myminio/versioned/ > expanded_ns.txt
+./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt
+
+./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+
+until $(./mc admin decom status myminio/ | grep -q Complete)
+do
+    echo "waiting for decom to finish..."
+    sleep 1
+done
+
+kill $pid
+
+(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log)&
+pid=$!
+
+sleep 2
+
+decom_user_count=$(./mc admin user list myminio/ | wc -l)
+decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $decom_user_count ]; then
+    echo "BUG: original user count differs after decommission"
+    exit 1
+fi
+
+if [ $policy_count -ne $decom_policy_count ]; then
+    echo "BUG: original policy count differs after decommission"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected versioning enabled after decommission"
+    exit 1
+fi
+
+./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected encryption enabled after expansion"
+    exit 1
+fi
+
+got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum)
+if [ "${expected_checksum}" != "${got_checksum}" ]; then
+    echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}"
+    exit 1
+fi
+
+./mc ls -r myminio/versioned > decommissioned_ns.txt
+./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt
+
+out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+kill $pid

docs/distributed/decom-encrypted.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+if [ -n "$TEST_DEBUG" ]; then
+    set -x
+fi
+
+pkill minio
+rm -rf /tmp/xl
+
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+	chmod +x mc
+fi
+
+export CI=true
+export MINIO_KMS_AUTO_ENCRYPTION=on
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+
+(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)&
+pid=$!
+
+sleep 2
+
+export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
+
+./mc admin user add myminio/ minio123 minio123
+./mc admin user add myminio/ minio12345 minio12345
+
+./mc admin policy add myminio/ rw ./docs/distributed/rw.json
+./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+
+./mc admin policy set myminio/ rw user=minio123
+./mc admin policy set myminio/ lake,rw user=minio12345
+
+./mc mb -l myminio/versioned
+
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+## Soft delete (creates delete markers)
+./mc rm -r --force myminio/versioned >/dev/null
+
+## mirror again to create another set of version on top
+./mc mirror internal myminio/versioned/ --quiet >/dev/null
+
+expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum)
+
+user_count=$(./mc admin user list myminio/ | wc -l)
+policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+kill $pid
+(minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
+pid=$!
+
+sleep 2
+
+expanded_user_count=$(./mc admin user list myminio/ | wc -l)
+expanded_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $expanded_user_count ]; then
+    echo "BUG: original user count differs from expanded setup"
+    exit 1
+fi
+
+if [ $policy_count -ne $expanded_policy_count ]; then
+    echo "BUG: original policy count  differs from expanded setup"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "expected versioning enabled after expansion"
+    exit 1
+fi
+
+./mc mirror cmd myminio/versioned/ --quiet >/dev/null
+
+./mc ls -r myminio/versioned/ > expanded_ns.txt
+./mc ls -r --versions myminio/versioned/ > expanded_ns_versions.txt
+
+./mc admin decom start myminio/ /tmp/xl/{1...10}/disk{0...1}
+
+until $(./mc admin decom status myminio/ | grep -q Complete)
+do
+    echo "waiting for decom to finish..."
+    sleep 1
+done
+
+kill $pid
+
+(minio server /tmp/xl/{11...30}/disk{0...3} 2>&1 >/dev/null)&
+pid=$!
+
+sleep 2
+
+decom_user_count=$(./mc admin user list myminio/ | wc -l)
+decom_policy_count=$(./mc admin policy list myminio/ | wc -l)
+
+if [ $user_count -ne $decom_user_count ]; then
+    echo "BUG: original user count differs after decommission"
+    exit 1
+fi
+
+if [ $policy_count -ne $decom_policy_count ]; then
+    echo "BUG: original policy count differs after decommission"
+    exit 1
+fi
+
+./mc version info myminio/versioned | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected versioning enabled after decommission"
+    exit 1
+fi
+
+./mc ls -r myminio/versioned > decommissioned_ns.txt
+./mc ls -r --versions myminio/versioned > decommissioned_ns_versions.txt
+
+out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum)
+if [ "${expected_checksum}" != "${got_checksum}" ]; then
+    echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}"
+    exit 1
+fi
+
+kill $pid

docs/distributed/decom.sh

@@ -1,16 +1,19 @@
 #!/bin/bash
 
-if [ -n "$DEBUG" ]; then
+if [ -n "$TEST_DEBUG" ]; then
     set -x
 fi
 
 pkill minio
 rm -rf /tmp/xl
 
-wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
-    chmod +x mc
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+	chmod +x mc
+fi
 
 export CI=true
+
 (minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null)&
 pid=$!
 
@@ -37,6 +40,8 @@ export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 ## mirror again to create another set of version on top
 ./mc mirror internal myminio/versioned/ --quiet >/dev/null
 
+expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum)
+
 user_count=$(./mc admin user list myminio/ | wc -l)
 policy_count=$(./mc admin policy list myminio/ | wc -l)
 
@@ -113,12 +118,20 @@ out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt)
 ret=$?
 if [ $ret -ne 0 ]; then
     echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
 fi
 
 out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt)
 ret=$?
 if [ $ret -ne 0 ]; then
     echo "BUG: expected no missing entries after decommission: $out"
+    exit 1
+fi
+
+got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum)
+if [ "${expected_checksum}" != "${got_checksum}" ]; then
+    echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}"
+    exit 1
 fi
 
 kill $pid

docs/gateway/nas.md

@@ -1,4 +1,4 @@
-# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) 
+# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
 
 > NAS gateway is deprecated and will be removed in future, no more fresh deployments are supported.
 
@@ -96,7 +96,7 @@ NAS gateway implementation allows symlinks on regular files.
 
 ### Behavior
 
-- For reads symlinks resolve to the file the symlink points to. 
+- For reads symlinks resolve to the file the symlink points to.
 - For deletes
   - Deleting a symlink deletes the symlink but not the real file to which the symlink points.
   - Deleting the real file a symlink points to automatically makes the dangling symlink invisible.

docs/iam/access-management-plugin.md

@@ -156,4 +156,3 @@ The following structure is also accepted:
 ```
 
 Any unmentioned JSON object keys in the above are ignored.
-

docs/iam/opa.md

@@ -1,4 +1,5 @@
 # OPA Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+
 OPA is a lightweight general-purpose policy engine that can be co-located with MinIO server, in this document we talk about how to use OPA HTTP API to authorize requests. It can be used with any type of credentials (STS based like OpenID or LDAP, regular IAM users or service accounts).
 
 OPA is enabled through MinIO's Access Management Plugin feature.
@@ -21,6 +22,7 @@ podman run -it \
 ### 2. Create a sample OPA Policy
 
 In another terminal, create a policy that allows root user all access and for all other users denies `PutObject`:
+
 ```sh
 cat > example.rego <<EOF
 package httpapi.authz
@@ -43,6 +45,7 @@ EOF
 ```
 
 Then load the policy via OPA's REST API.
+
 ```
 curl -X PUT --data-binary @example.rego \
   localhost:8181/v1/policies/putobject
@@ -78,4 +81,3 @@ mc cat foo/test/issue
 # 3. Attempt to upload an object as user `foo` - this will fail with a permissions error.
 mc cp /etc/issue myminio/test/issue2
 ```
-

docs/kms/README.md

@@ -123,13 +123,14 @@ Encrypted :
 ## Encrypted Private Key
 
 MinIO supports encrypted KES client private keys. Therefore, you can use
-an password-protected private keys for `MINIO_KMS_KES_KEY_FILE`. 
+an password-protected private keys for `MINIO_KMS_KES_KEY_FILE`.
 
 When using password-protected private keys for accessing KES you need to
 provide the password via:
+
 ```
 export MINIO_KMS_KES_KEY_PASSWORD=<your-password>
-``` 
+```
 
 Note that MinIO only supports encrypted private keys - not encrypted certificates.
 Certificates are no secrets and sent in plaintext as part of the TLS handshake.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-07-04T21-02-54Z
+  image: quay.io/minio/minio:RELEASE.2022-07-15T03-44-22Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/run-multi-site-ldap.sh

@@ -257,6 +257,9 @@ fi
 kill -9 ${site1_pid}
 # Update tag on minio2/newbucket when minio1 is down
 ./mc tag set minio2/newbucket "key=val2"
+# create a new bucket on minio2. This should replicate to minio1 after it comes online.
+./mc mb minio2/newbucket2
+
 # Restart minio1 instance
 minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
 sleep 10
@@ -267,4 +270,10 @@ if [ "${val}" != "val2" ]; then
     exit_1;
 fi
 
+# Test if bucket created when minio1 is down healed
+diff -q <(./mc ls minio1 | awk '{print $3}') <(./mc ls minio2 | awk '{print $3}') 1>/dev/null
+if  [ $? -ne 0 ]; then
+    echo "expected bucket to have replicated, exiting..."
+    exit_1;
+fi
 cleanup

docs/site-replication/run-multi-site-minio-idp.sh

@@ -317,7 +317,8 @@ fi
 kill -9 ${site1_pid}
 # Update tag on minio2/newbucket when minio1 is down
 ./mc tag set minio2/newbucket "key=val2"
-
+# create a new bucket on minio2. This should replicate to minio1 after it comes online.
+./mc mb minio2/newbucket2
 # Restart minio1 instance
 minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
 sleep 15
@@ -328,3 +329,9 @@ if [ "${val}" != "val2" ]; then
     echo "expected bucket tag to have replicated, exiting..."
     exit_1;
 fi
+# Test if bucket created when minio1 is down healed
+diff -q <(./mc ls minio1 | awk '{print $3}') <(./mc ls minio2 | awk '{print $3}') 1>/dev/null
+if  [ $? -ne 0 ]; then
+    echo "expected bucket to have replicated, exiting..."
+    exit_1;
+fi

docs/site-replication/run-multi-site-oidc.sh

@@ -242,6 +242,9 @@ fi
 kill -9 ${site1_pid}
 # Update tag on minio2/newbucket when minio1 is down
 ./mc tag set minio2/newbucket "key=val2"
+# create a new bucket on minio2. This should replicate to minio1 after it comes online.
+./mc mb minio2/newbucket2
+
 # Restart minio1 instance
 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 &
 sleep 10
@@ -251,3 +254,10 @@ if [ "${val}" != "val2" ]; then
     echo "expected bucket tag to have replicated, exiting..."
     exit_1;
 fi
+
+# Test if bucket created when minio1 is down healed
+diff -q <(./mc ls minio1 | awk '{print $3}') <(./mc ls minio2 | awk '{print $3}') 1>/dev/null
+if  [ $? -ne 0 ]; then
+    echo "expected bucket to have replicated, exiting..."
+    exit_1;
+fi

docs/sts/web-identity.md

@@ -10,7 +10,6 @@ By default, the temporary security credentials created by AssumeRoleWithWebIdent
 
 ## Configuring OpenID identity provider on MinIO
 
-
 Configuration can be performed via MinIO's standard configuration API (i.e. using `mc admin config set/get` commands) or equivalently via environment variables. For brevity we show only environment variables here:
 
 ```

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/inconshreveable/mousetrap v1.0.0
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.15.6
+	github.com/klauspost/compress v1.15.8
 	github.com/klauspost/cpuid/v2 v2.0.14
 	github.com/klauspost/pgzip v1.2.5
 	github.com/klauspost/readahead v1.4.0
@@ -47,9 +47,9 @@ require (
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
-	github.com/minio/kes v0.19.2
+	github.com/minio/kes v0.20.0
 	github.com/minio/madmin-go v1.4.3
-	github.com/minio/minio-go/v7 v7.0.30
+	github.com/minio/minio-go/v7 v7.0.31
 	github.com/minio/pkg v1.1.26
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
@@ -65,7 +65,7 @@ require (
 	github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
 	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.34.0
 	github.com/prometheus/procfs v0.7.3

go.sum

@@ -512,8 +512,8 @@ github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
-github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA=
+github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -618,8 +618,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX
 github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
-github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM=
-github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM=
+github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE=
+github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
 github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
 github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU=
 github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
@@ -629,8 +629,8 @@ github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77Z
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
-github.com/minio/minio-go/v7 v7.0.30 h1:Re+qlwA+LB3mgFGYbztVPzlEjKtGzRVV5Sk38np858k=
-github.com/minio/minio-go/v7 v7.0.30/go.mod h1:/sjRKkKIA75CKh1iu8E3qBy7ktBmCCDGII0zbXGwbUk=
+github.com/minio/minio-go/v7 v7.0.31 h1:zsJ3qPDeU3bC5UMVi9HJ4ED0lyEzrNd3iQguglZS5FE=
+github.com/minio/minio-go/v7 v7.0.31/go.mod h1:/sjRKkKIA75CKh1iu8E3qBy7ktBmCCDGII0zbXGwbUk=
 github.com/minio/pkg v1.1.20/go.mod h1:Xo7LQshlxGa9shKwJ7NzQbgW4s8T/Wc1cOStR/eUiMY=
 github.com/minio/pkg v1.1.26 h1:a8x4sHNBxCiHEkxZ/0EBTLqvV3nMtM2G/A6lXNfXN3U=
 github.com/minio/pkg v1.1.26/go.mod h1:z9PfmEI804KFkF6eY4LoGe8IDVvTCsYGVuaf58Dr0WI=
@@ -745,8 +745,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

helm/minio/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 4.0.2
+version: 4.0.5
 appVersion: RELEASE.2022-05-08T23-50-31Z
 keywords:
   - minio

helm/minio/templates/_helper_create_user.txt

@@ -8,6 +8,10 @@ MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
 MC="/usr/bin/mc --insecure"
 {{- end }}
 
+# AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters.
+# Special characters for example : ',",<,>,{,}
+MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp"
+
 # connectToMinio
 # Use a check-sleep-check loop to wait for MinIO service to be available
 connectToMinio() {
@@ -35,28 +39,36 @@ connectToMinio() {
   return 0
 }
 
-# checkUserExists ($username)
+# checkUserExists ()
 # Check if the user exists, by using the exit code of `mc admin user info`
 checkUserExists() {
-  USER=$1
-  CMD=$(${MC} admin user info myminio $USER > /dev/null 2>&1)
+  CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1)
   return $?
 }
 
-# createUser ($username, $password, $policy)
+# createUser ($policy)
 createUser() {
-  USER=$1
-  PASS=$2
-  POLICY=$3
-
+  POLICY=$1
+  #check accessKey_and_secretKey_tmp file
+  if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then
+    echo "credentials file does not exist"
+    return 1
+  fi
+  if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then
+    echo "credentials file is invalid"
+    rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
+    return 1
+  fi
   # Create the user if it does not exist
-  if ! checkUserExists $USER ; then
+  if ! checkUserExists ; then
+    USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP)
     echo "Creating user '$USER'"
-    ${MC} admin user add myminio $USER $PASS
+    cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio
   else
     echo "User '$USER' already exists."
   fi
-
+  #clean up credentials files.
+  rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
 
   # set policy for user
   if [ ! -z $POLICY -a $POLICY != " " ] ; then
@@ -79,10 +91,13 @@ connectToMinio $scheme
 {{ $global := . }}
 # Create the users
 {{- range .Values.users }}
+echo {{ tpl .accessKey $global }} > $MINIO_ACCESSKEY_SECRETKEY_TMP
 {{- if .existingSecret }}
-createUser {{ tpl .accessKey $global }} $(cat /config/secrets/{{ tpl .accessKey $global }}) {{ .policy }}
+cat /config/secrets/{{ tpl .secretKey $global }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP
+createUser {{ .policy }}
 {{ else }}
-createUser {{ tpl .accessKey $global }} {{ .secretKey }} {{ .policy }}
+echo {{ .secretKey }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP
+createUser {{ .policy }}
 {{- end }}
 {{- end }}
 {{- end }}

helm/minio/templates/console-service.yaml

@@ -35,7 +35,7 @@ spec:
 {{- if (and (eq .Values.consoleService.type "NodePort") ( .Values.consoleService.nodePort)) }}
       nodePort: {{ .Values.consoleService.nodePort }}
 {{- else }}
-      targetPort: {{ .Values.consoleService.port }}
+      targetPort: {{ .Values.minioConsolePort }}
 {{- end}}
 {{- if .Values.consoleService.externalIPs }}
   externalIPs:

helm/minio/templates/networkpolicy.yaml

@@ -16,8 +16,8 @@ spec:
       release: {{ .Release.Name }}
   ingress:
     - ports:
-        - port: {{ .Values.service.port }}
-        - port: {{ .Values.consoleService.port }}
+        - port: {{ .Values.minioAPIPort }}
+        - port: {{ .Values.minioConsolePort }}
       {{- if not .Values.networkPolicy.allowExternal }}
       from:
         - podSelector:

helm/minio/templates/service.yaml

@@ -36,7 +36,7 @@ spec:
 {{- if (and (eq .Values.service.type "NodePort") ( .Values.service.nodePort)) }}
       nodePort: {{ .Values.service.nodePort }}
 {{- else }}
-      targetPort: {{ .Values.service.port }}
+      targetPort: {{ .Values.minioAPIPort }}
 {{- end}}
 {{- if .Values.service.externalIPs }}
   externalIPs:

helm/minio/templates/servicemonitor.yaml

@@ -16,11 +16,21 @@ metadata:
     {{- if .Values.metrics.serviceMonitor.additionalLabels }}
 {{ toYaml .Values.metrics.serviceMonitor.additionalLabels | indent 4 }}
     {{- end }}
+{{- if .Values.metrics.serviceMonitor.annotations }}
+  annotations:
+{{ toYaml .Values.metrics.serviceMonitor.annotations | trimSuffix "\n" | indent 4 }}
+{{- end }}
 spec:
   endpoints:
     {{- if .Values.tls.enabled }}
     - port: https
       scheme: https
+      tlsConfig:
+        ca:
+          secret:
+            name: {{ .Values.tls.certSecret }}
+            key: {{ .Values.tls.publicCrt }}
+        serverName: {{ template "minio.fullname" . }}
     {{ else }}
     - port: http
       scheme: http

helm/minio/templates/statefulset.yaml

@@ -1,6 +1,7 @@
 {{- if eq .Values.mode "distributed" }}
 {{ $poolCount := .Values.pools | int }}
 {{ $nodeCount := .Values.replicas | int }}
+{{ $replicas := mul $poolCount $nodeCount }}
 {{ $drivesPerNode := .Values.drivesPerNode | int }}
 {{ $scheme := "http" }}
 {{- if .Values.tls.enabled }}
@@ -30,6 +31,7 @@ spec:
     - name: {{ $scheme }}
       port: {{ .Values.service.port }}
       protocol: TCP
+      targetPort: {{ .Values.minioAPIPort }}
   selector:
     app: {{ template "minio.name" . }}
     release: {{ .Release.Name }}
@@ -56,7 +58,7 @@ spec:
     type: {{ .Values.StatefulSetUpdate.updateStrategy }}
   podManagementPolicy: "Parallel"
   serviceName: {{ template "minio.fullname" . }}-svc
-  replicas: {{ mul $poolCount $nodeCount }}
+  replicas: {{ $replicas }}
   selector:
     matchLabels:
       app: {{ template "minio.name" . }}
@@ -167,6 +169,12 @@ spec:
     {{- with .Values.tolerations }}
       tolerations:
 {{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- if and (gt $replicas 1) (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor "19") }}
+    {{- with .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+{{ toYaml . | indent 8 }}
+    {{- end }}
     {{- end }}
       volumes:
         - name: minio-user