disallow EC:0 if not set during server startup (#17141)

Signed-off-by: Benjamin Perez <benjamin@bexsoft.net>

.github/markdown-lint-cfg.yaml

@@ -1,5 +0,0 @@
-# Config file for markdownlint-cli
-MD033:
-  allowed_elements:
-    - details
-    - summary

.github/workflows/markdown-lint.yaml

@@ -1,30 +0,0 @@
-name: Markdown Linter
-
-on:
-  pull_request:
-    branches:
-    - master
-
-# This ensures that previous jobs for the PR are canceled when the PR is
-# updated.
-concurrency: 
-  group: ${{ github.workflow }}-${{ github.head_ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  lint:
-    name: Lint all docs
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out code
-      uses: actions/checkout@v3
-
-    - name: Lint all docs
-      run: |
-        npm install -g markdownlint-cli
-        markdownlint --fix '**/*.md' \
-          --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
-          --disable MD013 MD040 MD051

.github/workflows/root-disable.yml

@@ -0,0 +1,34 @@
+name: Root lockdown tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.20.x]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Start root lockdown tests
+        run: |
+          make test-root-disable

Makefile

@@ -49,6 +49,10 @@ test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
 
+test-root-disable: install
+	@echo "Running minio root lockdown tests"
+	@env bash $(PWD)/buildscripts/disable-root.sh
+
 test-decom: install
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh

buildscripts/disable-root.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+set -x
+
+export MINIO_CI_CD=1
+killall -9 minio
+
+rm -rf ${HOME}/tmp/dist
+
+scheme="http"
+nr_servers=4
+
+addr="localhost"
+args=""
+for ((i=0;i<$[${nr_servers}];i++)); do
+    args="$args $scheme://$addr:$[9100+$i]/${HOME}/tmp/dist/path1/$i"
+done
+
+echo $args
+
+for ((i=0;i<$[${nr_servers}];i++)); do
+    (minio server --address ":$[9100+$i]" $args 2>&1 > /tmp/log$i.txt) &
+done
+
+sleep 10s
+
+if [ ! -f ./mc ]; then
+    wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc && \
+       chmod +x mc
+fi
+
+set +e
+
+export MC_HOST_minioadm=http://minioadmin:minioadmin@localhost:9100/
+
+./mc ls minioadm/
+
+./mc admin config set minioadm/ api root_access=off
+
+sleep 3s # let things settle a little
+
+./mc ls minioadm/
+if [ $? -eq 0 ]; then
+    echo "listing succeeded, 'minioadmin' was not disabled"
+    exit 1
+fi
+
+set -e
+
+killall -9 minio
+
+export MINIO_API_ROOT_ACCESS=on
+for ((i=0;i<$[${nr_servers}];i++)); do
+    (minio server --address ":$[9100+$i]" $args 2>&1 > /tmp/log$i.txt) &
+done
+
+set +e
+
+./mc ls minioadm/
+if [ $? -ne 0 ]; then
+    echo "listing failed, 'minioadmin' should be enabled"
+    exit 1
+fi
+
+killall -9 minio
+
+rm -rf /tmp/multisitea/
+rm -rf /tmp/multisiteb/
+
+echo "Setup site-replication and then disable root credentials"
+
+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+
+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+
+sleep 20s
+
+export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001
+export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004
+
+./mc admin replicate add sitea siteb
+
+./mc admin user add sitea foobar foo12345
+
+./mc admin policy attach sitea/ consoleAdmin --user=foobar
+
+./mc admin user info siteb foobar
+
+killall -9 minio
+
+echo "turning off root access, however site replication must continue"
+export MINIO_API_ROOT_ACCESS=off
+
+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+
+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+      "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+
+sleep 20s
+
+export MC_HOST_sitea=http://foobar:foo12345@127.0.0.1:9001
+export MC_HOST_siteb=http://foobar:foo12345@127.0.0.1:9004
+
+./mc admin user add sitea foobar-admin foo12345
+
+sleep 2s
+
+./mc admin user info siteb foobar-admin

cmd/admin-handlers-users-race_test.go

@@ -32,7 +32,7 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	minio "github.com/minio/minio-go/v7"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {

cmd/admin-handlers-users.go

@@ -44,7 +44,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -62,6 +62,13 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// When the user is root credential you are not allowed to
+	// remove the root user. Also you cannot delete yourself.
+	if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+		return
+	}
+
 	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -239,6 +246,26 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+
+	// Reject if the group add and remove are temporary credentials, or root credential.
+	for _, member := range updReq.Members {
+		ok, _, err := globalIAMSys.IsTempUser(member)
+		if err != nil && err != errNoSuchUser {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if ok {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if member == globalActiveCred.AccessKey {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+	}
+
 	var updatedAt time.Time
 	if updReq.IsRemove {
 		updatedAt, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
@@ -374,7 +401,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -383,9 +410,9 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
 
-	// This API is not allowed to lookup master access key user status
-	if accessKey == globalActiveCred.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+	// you cannot enable or disable yourself.
+	if accessKey == creds.AccessKey {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
 		return
 	}
 
@@ -1627,6 +1654,12 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if entityName == globalActiveCred.AccessKey {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
 	}
 
 	// Validate that user or group exists.
@@ -1771,6 +1804,13 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			return
 		}
 
+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if userOrGroup == globalActiveCred.AccessKey {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+
 		// Validate that user exists.
 		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
 			_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
@@ -2056,13 +2096,15 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			userAccounts := make(map[string]madmin.AddOrUpdateUserReq)
 			for u, uid := range userIdentities {
-				status := madmin.AccountDisabled
-				if uid.Credentials.IsValid() {
-					status = madmin.AccountEnabled
-				}
 				userAccounts[u] = madmin.AddOrUpdateUserReq{
 					SecretKey: uid.Credentials.SecretKey,
-					Status:    status,
+					Status: func() madmin.AccountStatus {
+						// Export current credential status
+						if uid.Credentials.Status == auth.AccountOff {
+							return madmin.AccountDisabled
+						}
+						return madmin.AccountEnabled
+					}(),
 				}
 			}
 			userData, err := json.Marshal(userAccounts)
@@ -2101,6 +2143,10 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			svcAccts := make(map[string]madmin.SRSvcAccCreate)
 			for user, acc := range serviceAccounts {
+				if user == siteReplicatorSvcAcc {
+					// skip site-replication service account.
+					continue
+				}
 				claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
@@ -2461,12 +2507,13 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					continue
 				}
 				opts := newServiceAccountOpts{
-					accessKey:     user,
-					secretKey:     svcAcctReq.SecretKey,
-					sessionPolicy: sp,
-					claims:        svcAcctReq.Claims,
-					comment:       svcAcctReq.Comment,
-					expiration:    svcAcctReq.Expiration,
+					accessKey:                  user,
+					secretKey:                  svcAcctReq.SecretKey,
+					sessionPolicy:              sp,
+					claims:                     svcAcctReq.Claims,
+					comment:                    svcAcctReq.Comment,
+					expiration:                 svcAcctReq.Expiration,
+					allowSiteReplicatorAccount: false,
 				}
 
 				// In case of LDAP we need to resolve the targetUser to a DN and

cmd/admin-handlers.go

@@ -1896,6 +1896,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 			OfflineDisks:     offlineDisks.Sum(),
 			StandardSCParity: backendInfo.StandardSCParity,
 			RRSCParity:       backendInfo.RRSCParity,
+			TotalSets:        backendInfo.TotalSets,
+			DrivesPerSet:     backendInfo.DrivesPerSet,
 		}
 
 		if poolsInfoEnabled {

cmd/admin-handlers_test.go

@@ -479,8 +479,8 @@ func TestTopLockEntries(t *testing.T) {
 		if len(exp) != len(got) {
 			return 0, false
 		}
-		sort.Sort(byResourceUID{exp})
-		sort.Sort(byResourceUID{got})
+		sort.Slice(exp, byResourceUID{exp}.Less)
+		sort.Slice(got, byResourceUID{got}.Less)
 		// printEntries(exp)
 		// printEntries(got)
 		for i, e := range exp {

cmd/admin-heal-ops.go

@@ -784,6 +784,11 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 }
 
 func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+	if h.clientToken == bgHealingUUID {
+		// For background heal do nothing.
+		return nil
+	}
+
 	if err := h.healDiskMeta(objAPI); err != nil {
 		return err
 	}

cmd/admin-server-info.go

@@ -87,7 +87,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	}
 
 	props := madmin.ServerProperties{
-		State:    string(madmin.ItemInitializing),
 		Endpoint: addr,
 		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
 		Version:  Version,
@@ -146,7 +145,8 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {
-		props.State = string(madmin.ItemOffline)
+		props.State = string(madmin.ItemInitializing)
+		props.Disks = getOfflineDisks("", globalEndpoints)
 	}
 
 	return props

cmd/api-response.go

@@ -743,7 +743,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
 func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
-	cs := oi.decryptChecksums()
+	cs := oi.decryptChecksums(0)
 	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,

cmd/auth-handler_test.go

@@ -375,8 +375,6 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
-
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
 		t.Fatalf("unable create credential, %s", err)
@@ -384,6 +382,8 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	globalActiveCred = creds
 
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
 		req     *http.Request
@@ -464,9 +464,8 @@ func TestValidateAdminSignature(t *testing.T) {
 	}
 
 	initAllSubsystems(ctx)
-	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	initConfigSubsystem(ctx, objLayer)
 
 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {
@@ -474,6 +473,8 @@ func TestValidateAdminSignature(t *testing.T) {
 	}
 	globalActiveCred = creds
 
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+
 	testCases := []struct {
 		AccessKey string
 		SecretKey string

cmd/batch-handlers.go

@@ -48,11 +48,11 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/minio/pkg/wildcard"
+	"github.com/minio/pkg/workers"
 	"gopkg.in/yaml.v2"
 )
 
@@ -174,6 +174,7 @@ type BatchJobReplicateResourceType string
 func (t BatchJobReplicateResourceType) Validate() error {
 	switch t {
 	case BatchJobReplicateResourceMinIO:
+	case BatchJobReplicateResourceS3:
 	default:
 		return errInvalidArgument
 	}
@@ -183,6 +184,8 @@ func (t BatchJobReplicateResourceType) Validate() error {
 // Different types of batch jobs..
 const (
 	BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
+	BatchJobReplicateResourceS3    BatchJobReplicateResourceType = "s3"
+
 	// add future targets
 )
 
@@ -297,10 +300,13 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 
 	versioned := globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject)
-
+	versionID := srcObjInfo.VersionID
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		versionID = ""
+	}
 	if srcObjInfo.DeleteMarker {
 		_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
-			VersionID:          srcObjInfo.VersionID,
+			VersionID:          versionID,
 			VersionSuspended:   versionSuspended,
 			Versioned:          versioned,
 			MTime:              srcObjInfo.ModTime,
@@ -318,6 +324,9 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 		PreserveETag:     srcObjInfo.ETag,
 		UserDefined:      srcObjInfo.UserDefined,
 	}
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		opts.VersionID = ""
+	}
 	if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) {
 		opts.ServerSideEncryption = encrypt.NewSSE()
 	}
@@ -358,7 +367,9 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 	if r.Target.Prefix != "" {
 		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
 	}
-
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		opts.VersionID = ""
+	}
 	var uploadedParts []CompletePart
 	res, err := api.NewMultipartUpload(context.Background(), tgtBucket, tgtObject, opts)
 	if err != nil {
@@ -539,19 +550,47 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
-
+		// one of source/target is s3, skip delete marker and all versions under the same object name.
+		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
+		minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
 		ctx, cancel := context.WithCancel(ctx)
 		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{
 			Prefix:       r.Source.Prefix,
-			WithVersions: true,
+			WithVersions: minioSrc,
 			Recursive:    true,
 			WithMetadata: true,
 		})
+		prevObj := ""
+		skipReplicate := false
+
 		for obj := range objInfoCh {
 			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
+			if !minioSrc {
+				oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, minio.StatObjectOptions{})
+				if err == nil {
+					oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
+				} else {
+					if isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) ||
+						isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+						continue
+					}
+					logger.LogIf(ctx, err)
+					cancel()
+					return err
+				}
+			}
 			if skip(oi) {
 				continue
 			}
+			if obj.Key != prevObj {
+				prevObj = obj.Key
+				// skip replication of delete marker and all versions under the same object name if one of source or target is s3.
+				skipReplicate = obj.IsDeleteMarker && s3Type
+			}
+			if skipReplicate {
+				continue
+			}
+
 			wk.Take()
 			go func() {
 				defer wk.Give()
@@ -646,9 +685,10 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 	tgtBucket := r.Target.Bucket
 	tgtPrefix := r.Target.Prefix
 	srcObject := srcObjInfo.Name
+	s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
 	if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() {
-		if retry {
+		if retry && !s3Type {
 			if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), miniogo.StatObjectOptions{
 				VersionID: srcObjInfo.VersionID,
 				Internal: miniogo.AdvancedGetOptions{
@@ -664,7 +704,10 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 		if srcObjInfo.VersionPurgeStatus.Empty() {
 			dmVersionID = srcObjInfo.VersionID
 		}
-
+		if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+			dmVersionID = ""
+			versionID = ""
+		}
 		return c.RemoveObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), miniogo.RemoveObjectOptions{
 			VersionID: versionID,
 			Internal: miniogo.AdvancedRemoveOptions{
@@ -676,7 +719,7 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 		})
 	}
 
-	if retry { // when we are retrying avoid copying if necessary.
+	if retry && !s3Type { // when we are retrying avoid copying if necessary.
 		gopts := miniogo.GetObjectOptions{}
 		if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
 			return err
@@ -710,7 +753,9 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 	if err != nil {
 		return err
 	}
-
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		putOpts.Internal = miniogo.AdvancedPutOptions{}
+	}
 	if objInfo.isMultipart() {
 		if err := replicateObjectWithMultipart(ctx, c, tgtBucket, pathJoin(tgtPrefix, objInfo.Name), rd, objInfo, putOpts); err != nil {
 			return err
@@ -1021,6 +1066,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// None of the provided metadata filters match skip the object.
 			return false
 		}
+		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
+		if (r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest {
+			return false
+		}
 
 		return true
 	}
@@ -1059,6 +1108,8 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		attempts := attempts
 
 		ctx, cancel := context.WithCancel(ctx)
+		// one of source/target is s3, skip delete marker and all versions under the same object name.
+		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
 		results := make(chan ObjectInfo, 100)
 		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, ObjectOptions{
@@ -1070,8 +1121,18 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			return err
 		}
 
+		prevObj := ""
+
+		skipReplicate := false
 		for result := range results {
 			result := result
+			if result.Name != prevObj {
+				prevObj = result.Name
+				skipReplicate = result.DeleteMarker && s3Type
+			}
+			if skipReplicate {
+				continue
+			}
 			wk.Take()
 			go func() {
 				defer wk.Give()
@@ -1265,8 +1326,10 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		}
 		return err
 	}
+	// if both source and target are minio instances
+	minioType := r.Target.Type == BatchJobReplicateResourceMinIO && r.Source.Type == BatchJobReplicateResourceMinIO
 	// If source has versioning enabled, target must have versioning enabled
-	if (info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal) {
+	if minioType && ((info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal)) {
 		return batchReplicationJobError{
 			Code: "InvalidBucketState",
 			Description: fmt.Sprintf("The source '%s' has versioning enabled, target '%s' must have versioning enabled",
@@ -1355,9 +1418,10 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
 		return putOpts, err
 	}
 	putOpts.Internal = miniogo.AdvancedPutOptions{
-		SourceVersionID: objInfo.VersionID,
-		SourceMTime:     objInfo.ModTime,
-		SourceETag:      objInfo.ETag,
+		SourceVersionID:    objInfo.VersionID,
+		SourceMTime:        objInfo.ModTime,
+		SourceETag:         objInfo.ETag,
+		ReplicationRequest: true,
 	}
 	return putOpts, nil
 }

cmd/batch-rotate.go

@@ -38,9 +38,9 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/env"
 	"github.com/minio/pkg/wildcard"
+	"github.com/minio/pkg/workers"
 )
 
 // keyrotate:

cmd/bucket-handlers.go

@@ -51,9 +51,9 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 const (
@@ -540,6 +540,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			oss[index].SetTransitionState(goi.TransitionedObject)
 		}
 
+		// All deletes on directory objects needs to be for `nullVersionID`
+		if isDirObject(object.ObjectName) && object.VersionID == "" {
+			object.VersionID = nullVersionID
+		}
+
 		if replicateDeletes {
 			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectV: ObjectV{
@@ -1324,14 +1329,6 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	if globalDNSConfig != nil {
-		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
-		}
-	}
-
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
@@ -1345,15 +1342,18 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				apiErr.Description = "The bucket you tried to delete is not empty. You must delete all versions in the bucket."
 			}
 		}
-		if globalDNSConfig != nil {
-			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
-			}
-		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
+
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
 

cmd/bucket-handlers_test.go

@@ -657,13 +657,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 ) {
 	var err error
 
-	contentBytes := []byte("hello")
 	sha256sum := ""
 	var objectNames []string
 	for i := 0; i < 10; i++ {
+		contentBytes := []byte("hello")
 		objectName := "test-object-" + strconv.Itoa(i)
 		if i == 0 {
 			objectName += "/"
+			contentBytes = []byte{}
 		}
 		// uploading the object.
 		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
@@ -676,6 +677,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		objectNames = append(objectNames, objectName)
 	}
 
+	contentBytes := []byte("hello")
 	for _, name := range []string{"private/object", "public/object"} {
 		// Uploading the object with retention enabled
 		_, err = obj.PutObject(GlobalContext, bucketName, name, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
@@ -745,8 +747,13 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 	deletedObjects := make([]DeletedObject, len(requestList[0].Objects))
 	for i := range requestList[0].Objects {
+		var vid string
+		if isDirObject(requestList[0].Objects[i].ObjectName) {
+			vid = nullVersionID
+		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[0].Objects[i].ObjectName,
+			VersionID:  vid,
 		}
 	}
 
@@ -756,9 +763,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	successRequest1 := encodeResponse(requestList[1])
 
 	deletedObjects = make([]DeletedObject, len(requestList[1].Objects))
-	for i := range requestList[0].Objects {
+	for i := range requestList[1].Objects {
+		var vid string
+		if isDirObject(requestList[0].Objects[i].ObjectName) {
+			vid = nullVersionID
+		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[1].Objects[i].ObjectName,
+			VersionID:  vid,
 		}
 	}
 

cmd/bucket-lifecycle.go

@@ -41,8 +41,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
-	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/workers"
 )
 
 const (
@@ -96,9 +96,8 @@ func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
 }
 
 type expiryTask struct {
-	objInfo        ObjectInfo
-	versionExpiry  bool
-	restoredObject bool
+	objInfo ObjectInfo
+	event   lifecycle.Event
 }
 
 type expiryState struct {
@@ -121,22 +120,22 @@ func (es *expiryState) close() {
 }
 
 // enqueueByDays enqueues object versions expired by days for expiry.
-func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersion bool) {
+func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byDaysCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}:
+	case es.byDaysCh <- expiryTask{objInfo: oi, event: event}:
 	default:
 	}
 }
 
 // enqueueByNewerNoncurrent enqueues object versions expired by
 // NewerNoncurrentVersions limit for expiry.
-func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}:
 	default:
 	}
 }
@@ -171,9 +170,9 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 			go func(t expiryTask) {
 				defer ewk.Give()
 				if t.objInfo.TransitionedObject.Status != "" {
-					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.event)
 				} else {
-					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.event)
 				}
 			}(t)
 		}
@@ -184,7 +183,7 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 			nwk.Take()
 			go func(t newerNoncurrentTask) {
 				defer nwk.Give()
-				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions, t.event)
 			}(t)
 		}
 		nwk.Wait()
@@ -196,11 +195,12 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 type newerNoncurrentTask struct {
 	bucket   string
 	versions []ObjectToDelete
+	event    lifecycle.Event
 }
 
 type transitionTask struct {
-	tier    string
 	objInfo ObjectInfo
+	event   lifecycle.Event
 }
 
 type transitionState struct {
@@ -218,10 +218,10 @@ type transitionState struct {
 	lastDayStats map[string]*lastDayTierStats
 }
 
-func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event) {
 	select {
 	case <-t.ctx.Done():
-	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
+	case t.transitionCh <- transitionTask{objInfo: oi, event: event}:
 	default:
 	}
 }
@@ -274,9 +274,9 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
-				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
-					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.event); err != nil {
+				logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
+					task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
 					TotalSize:   uint64(task.objInfo.Size),
@@ -285,7 +285,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				if task.objInfo.IsLatest {
 					ts.NumObjects = 1
 				}
-				t.addLastDayStats(task.tier, ts)
+				t.addLastDayStats(task.event.StorageClass, ts)
 			}
 			atomic.AddInt32(&t.activeTasks, -1)
 		}
@@ -358,89 +358,76 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 // This is to be called after a successful upload of an object (version).
 func enqueueTransitionImmediate(obj ObjectInfo) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
-		event := lc.Eval(obj.ToLifecycleOpts())
-		switch event.Action {
+		switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj, event.StorageClass)
+			globalTransitionState.queueTransitionTask(obj, event)
 		}
 	}
 }
 
-// expireAction represents different actions to be performed on expiry of a
-// restored/transitioned object
-type expireAction int
-
-const (
-	// ignore the zero value
-	_ expireAction = iota
-	// expireObj indicates expiry of 'regular' transitioned objects.
-	expireObj
-	// expireRestoredObj indicates expiry of restored objects.
-	expireRestoredObj
-)
-
 // expireTransitionedObject handles expiry of transitioned/restored objects
 // (versions) in one of the following situations:
 //
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
-func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
+func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, lcEvent lifecycle.Event) error {
 	traceFn := globalLifecycleSys.trace(*oi)
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
 	opts.Expiration = ExpirationOptions{Expire: true}
-	switch action {
-	case expireObj:
-		// When an object is past expiry or when a transitioned object is being
-		// deleted, 'mark' the data in the remote tier for delete.
-		entry := jentry{
-			ObjName:   oi.TransitionedObject.Name,
-			VersionID: oi.TransitionedObject.VersionID,
-			TierName:  oi.TransitionedObject.Tier,
-		}
-		if err := globalTierJournal.AddEntry(entry); err != nil {
-			logger.LogIf(ctx, err)
-			return err
-		}
-		// Delete metadata on source, now that data in remote tier has been
-		// marked for deletion.
-		if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-			logger.LogIf(ctx, err)
-			return err
-		}
-
-		// Send audit for the lifecycle delete operation
-		defer auditLogLifecycle(ctx, *oi, ILMExpiry, traceFn)
-
-		eventName := event.ObjectRemovedDelete
-		if lcOpts.DeleteMarker {
-			eventName = event.ObjectRemovedDeleteMarkerCreated
-		}
-		objInfo := ObjectInfo{
-			Name:         oi.Name,
-			VersionID:    lcOpts.VersionID,
-			DeleteMarker: lcOpts.DeleteMarker,
-		}
-		// Notify object deleted event.
-		sendEvent(eventArgs{
-			EventName:  eventName,
-			BucketName: oi.Bucket,
-			Object:     objInfo,
-			UserAgent:  "Internal: [ILM-Expiry]",
-			Host:       globalLocalNodeName,
-		})
-
-	case expireRestoredObj:
+	tags := auditLifecycleTags(lcEvent)
+	if lcEvent.Action.DeleteRestored() {
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
 		opts.Transition.ExpireRestored = true
 		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+		if err == nil {
+			// TODO consider including expiry of restored object to events we
+			// notify.
+			auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+		}
 		return err
-	default:
-		return fmt.Errorf("Unknown expire action %v", action)
 	}
+	// When an object is past expiry or when a transitioned object is being
+	// deleted, 'mark' the data in the remote tier for delete.
+	entry := jentry{
+		ObjName:   oi.TransitionedObject.Name,
+		VersionID: oi.TransitionedObject.VersionID,
+		TierName:  oi.TransitionedObject.Tier,
+	}
+	if err := globalTierJournal.AddEntry(entry); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	// Delete metadata on source, now that data in remote tier has been
+	// marked for deletion.
+	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+
+	// Send audit for the lifecycle delete operation
+	defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+
+	eventName := event.ObjectRemovedDelete
+	if lcOpts.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+	objInfo := ObjectInfo{
+		Name:         oi.Name,
+		VersionID:    lcOpts.VersionID,
+		DeleteMarker: lcOpts.DeleteMarker,
+	}
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: oi.Bucket,
+		Object:     objInfo,
+		UserAgent:  "Internal: [ILM-Expiry]",
+		Host:       globalLocalNodeName,
+	})
 
 	return nil
 }
@@ -460,13 +447,14 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lcEvent lifecycle.Event) error {
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,
-			Tier:   tier,
+			Tier:   lcEvent.StorageClass,
 			ETag:   oi.ETag,
 		},
+		LifecycleEvent:   lcEvent,
 		VersionID:        oi.VersionID,
 		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
 		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
@@ -878,3 +866,35 @@ func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
 		TransitionStatus: oi.TransitionedObject.Status,
 	}
 }
+
+func auditLifecycleTags(event lifecycle.Event) map[string]interface{} {
+	const (
+		ilmAction                  = "ilm-action"
+		ilmDue                     = "ilm-due"
+		ilmRuleID                  = "ilm-rule-id"
+		ilmTier                    = "ilm-tier"
+		ilmNewerNoncurrentVersions = "ilm-newer-noncurrent-versions"
+		ilmNoncurrentDays          = "ilm-noncurrent-days"
+	)
+	tags := make(map[string]interface{}, 4)
+	tags[ilmAction] = event.Action.String()
+	tags[ilmRuleID] = event.RuleID
+
+	if !event.Due.IsZero() {
+		tags[ilmDue] = event.Due
+	}
+
+	// rule with Transition/NoncurrentVersionTransition in effect
+	if event.StorageClass != "" {
+		tags[ilmTier] = event.StorageClass
+	}
+
+	// rule with NewernoncurrentVersions in effect
+	if event.NewerNoncurrentVersions > 0 {
+		tags[ilmNewerNoncurrentVersions] = event.NewerNoncurrentVersions
+	}
+	if event.NoncurrentDays > 0 {
+		tags[ilmNoncurrentDays] = event.NoncurrentDays
+	}
+	return tags
+}

cmd/bucket-metadata-sys.go

@@ -36,8 +36,8 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.

cmd/bucket-policy-handlers.go

@@ -87,7 +87,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, APIError{
-			Code:           "ErrMalformedXML",
+			Code:           "MalformedPolicy",
 			HTTPStatusCode: http.StatusBadRequest,
 			Description:    err.Error(),
 		}, r.URL)

cmd/common-main.go

@@ -59,7 +59,6 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -133,16 +132,6 @@ func init() {
 		}
 	}()
 
-	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
-		PassHost:     true,
-		RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
-		Logger: func(err error) {
-			if err != nil && !errors.Is(err, context.Canceled) {
-				logger.LogIf(GlobalContext, err)
-			}
-		},
-	})
-
 	console.SetColor("Debug", fcolor.New())
 
 	gob.Register(StorageErr(""))
@@ -187,7 +176,7 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// pass the console subpath configuration
-	if value := env.Get(config.EnvMinIOBrowserRedirectURL, ""); value != "" {
+	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
 		if subPath != SlashSeparator {
 			os.Setenv("CONSOLE_SUBPATH", subPath)
@@ -208,6 +197,11 @@ func minioConfigToConsoleFeatures() {
 	if globalIAMSys.LDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
+	// Handle animation in welcome page
+	if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
+		os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
+	}
+
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
 
@@ -632,7 +626,7 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
 	if globalBrowserEnabled {
-		if redirectURL := env.Get(config.EnvMinIOBrowserRedirectURL, ""); redirectURL != "" {
+		if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
 			u, err := xnet.ParseHTTPURL(redirectURL)
 			if err != nil {
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value in environment variable")
@@ -777,6 +771,7 @@ func handleCommonEnvVars() {
 			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
+		globalCredViaEnv = true
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}

cmd/config-encrypted.go

@@ -1,215 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"path"
-	"time"
-	"unicode/utf8"
-
-	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
-	etcd "go.etcd.io/etcd/client/v3"
-)
-
-func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
-	encrypted, err := checkBackendEncrypted(objAPI)
-	if err != nil {
-		return fmt.Errorf("Unable to encrypt config %w", err)
-	}
-	if !encrypted {
-		return nil
-	}
-	if err = migrateConfigPrefixToEncrypted(objAPI); err != nil {
-		return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err)
-	}
-	return nil
-}
-
-const backendEncryptedFile = "backend-encrypted"
-
-var backendEncryptedMigrationComplete = []byte("encrypted")
-
-func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
-	bootstrapTrace("check if etcd backend is encrypted")
-	data, err := readKeyEtcd(ctx, client, backendEncryptedFile)
-	if err != nil && err != errConfigNotFound {
-		return false, err
-	}
-	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
-}
-
-func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
-	bootstrapTrace("check if the config backend is encrypted")
-	data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
-	if err != nil && err != errConfigNotFound {
-		return false, err
-	}
-	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
-}
-
-func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
-	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
-	if err != nil {
-		return err
-	}
-
-	// If backend doesn't have this file means we have already
-	// attempted then migration
-	if !encrypted {
-		return nil
-	}
-
-	bootstrapTrace("encrypt etcd config")
-
-	if GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat(ctx)
-		if err != nil {
-			return err
-		}
-		logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name))
-	}
-
-	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
-	defer cancel()
-
-	r, err := client.Get(listCtx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
-	if err != nil {
-		return err
-	}
-
-	for _, kv := range r.Kvs {
-		data, err := readKeyEtcd(ctx, client, string(kv.Key))
-		if err == errConfigNotFound { // Perhaps not present or someone deleted it.
-			continue
-		}
-		if err != nil {
-			return err
-		}
-
-		if !utf8.Valid(data) {
-			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-			if err != nil {
-				if GlobalKMS != nil {
-					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-						minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
-					})
-					if err != nil {
-						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-							minioMetaBucket: string(kv.Key),
-						})
-						if err != nil {
-							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-						}
-					}
-				} else {
-					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-				}
-			}
-			data = pdata
-		}
-
-		if GlobalKMS != nil {
-			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
-			})
-			if err != nil {
-				return err
-			}
-		}
-
-		if err = saveKeyEtcd(ctx, client, string(kv.Key), data); err != nil {
-			return err
-		}
-	}
-
-	if GlobalKMS != nil {
-		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
-	}
-
-	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
-}
-
-func migrateConfigPrefixToEncrypted(objAPI ObjectLayer) error {
-	bootstrapTrace("migrating config prefix to encrypted")
-	if GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat(context.Background())
-		if err != nil {
-			return err
-		}
-		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
-	}
-
-	results := make(chan ObjectInfo)
-	if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
-		return err
-	}
-
-	for obj := range results {
-		data, err := readConfig(GlobalContext, objAPI, obj.Name)
-		if err != nil {
-			return err
-		}
-
-		if !utf8.Valid(data) {
-			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-			if err != nil {
-				if GlobalKMS != nil {
-					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-						minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
-					})
-					if err != nil {
-						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-							minioMetaBucket: obj.Name,
-						})
-						if err != nil {
-							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-						}
-					}
-				} else {
-					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-				}
-			}
-			data = pdata
-		}
-
-		if GlobalKMS != nil {
-			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-				obj.Bucket: path.Join(obj.Bucket, obj.Name),
-			})
-			if err != nil {
-				return err
-			}
-		}
-
-		if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
-			return err
-		}
-	}
-
-	if GlobalKMS != nil {
-		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
-	}
-
-	return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
-}

cmd/config-migrate.go

@@ -18,16 +18,15 @@
 package cmd
 
 import (
-	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"unicode/utf8"
 
-	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
@@ -39,7 +38,6 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/event/target"
-	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/target/http"
 	xnet "github.com/minio/pkg/net"
@@ -2471,284 +2469,97 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	return saveServerConfig(GlobalContext, objAPI, config)
 }
 
-// Migrates '.minio.sys/config.json' to v33.
-func migrateMinioSysConfig(objAPI ObjectLayer) error {
-	bootstrapTrace("migrate .minio.sys/config/config.json to latest version")
+func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
-	// Check if the config version is latest, if not migrate.
-	ok, _, err := checkConfigVersion(objAPI, configFile, "33")
-	if err != nil {
-		return err
-	}
-	if ok {
-		return nil
+	configFiles := []string{
+		getConfigFile(),
+		getConfigFile() + ".deprecated",
+		configFile,
 	}
 
-	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV30ToV31MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV31ToV32MinioSys(objAPI); err != nil {
-		return err
-	}
-	return migrateV32ToV33MinioSys(objAPI)
-}
+	newServerCfg := func() (config.Config, error) {
+		// Initialize server config.
+		srvCfg := newServerConfig()
 
-func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
-	data, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		return false, nil, err
+		return srvCfg, saveServerConfig(ctx, objAPI, srvCfg)
 	}
 
-	if !utf8.Valid(data) {
-		if GlobalKMS != nil {
-			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: path.Join(minioMetaBucket, configFile),
-			})
-		}
+	var data []byte
+	var err error
 
-		if GlobalKMS == nil && err != nil {
-			data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-			if err != nil {
-				if err == madmin.ErrMaliciousData || err == madmin.ErrUnexpectedHeader {
-					return false, nil, config.ErrInvalidConfigDecryptionKey(nil)
-				}
-				return false, nil, err
+	cfg := &serverConfigV33{}
+	for _, cfgFile := range configFiles {
+		if _, err = Load(cfgFile, cfg); err != nil {
+			if !osIsNotExist(err) && !osIsPermission(err) {
+				return nil, err
 			}
+			continue
+		}
+		data, _ = json.Marshal(cfg)
+		break
+	}
+	if osIsPermission(err) {
+		logger.Info("Older config found but is not readable %s, proceeding to read config from other places", err)
+	}
+	if osIsNotExist(err) || osIsPermission(err) || len(data) == 0 {
+		data, err = readConfig(GlobalContext, objAPI, configFile)
+		if err != nil {
+			// when config.json is not found, then we freshly initialize.
+			if errors.Is(err, errConfigNotFound) {
+				return newServerCfg()
+			}
+			return nil, err
+		}
+
+		data, err = decryptData(data, configFile)
+		if err != nil {
+			return nil, err
+		}
+
+		newCfg, err := readServerConfig(GlobalContext, objAPI, data)
+		if err == nil {
+			return newCfg, nil
+		}
+
+		// Read older `.minio.sys/config/config.json`, if not
+		// possible just fail.
+		if err = json.Unmarshal(data, cfg); err != nil {
+			// Unable to parse old JSON simply re-initialize a new one.
+			return newServerCfg()
 		}
 	}
 
-	if version == "kvs" {
-		// Check api config values are present.
-		var vcfg struct {
-			API struct {
-				E []struct {
-					Key   string `json:"key"`
-					Value string `json:"value"`
-				} `json:"_"`
-			} `json:"api"`
+	if !globalCredViaEnv && cfg.Credential.IsValid() {
+		// Preserve older credential if we do not have
+		// root credentials set via environment variable.
+		globalActiveCred = cfg.Credential
+	}
+
+	// Init compression config. For future migration, Compression config needs to be copied over from previous version.
+	switch cfg.Version {
+	case "29":
+		// V29 -> V30
+		cfg.Compression.Enabled = false
+		cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
+		cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
+	case "30":
+		// V30 -> V31
+		cfg.OpenID = openid.Config{}
+		cfg.Policy.OPA = opa.Args{
+			URL:       &xnet.URL{},
+			AuthToken: "",
 		}
-		if err = json.Unmarshal(data, &vcfg); err != nil {
-			return false, nil, nil
-		}
-		return len(vcfg.API.E) > 0, data, nil
-	}
-	var versionConfig struct {
-		Version string `json:"version"`
-	}
-
-	vcfg := &versionConfig
-	if err = json.Unmarshal(data, vcfg); err != nil {
-		return false, nil, err
-	}
-	return vcfg.Version == version, data, nil
-}
-
-func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	ok, data, err := checkConfigVersion(objAPI, configFile, "27")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV28{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "28"
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
-	return nil
-}
-
-func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "28")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV29{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "29"
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
-	return nil
-}
-
-func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "29")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV30{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "30"
-	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
-	cfg.Compression.Enabled = false
-	cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
-	cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
-
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
-	return nil
-}
-
-func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "30")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV31{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "31"
-	cfg.OpenID = openid.Config{}
-
-	cfg.Policy.OPA = opa.Args{
-		URL:       &xnet.URL{},
-		AuthToken: "",
-	}
-
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
-	return nil
-}
-
-func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "31")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV32{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "32"
-	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
-	cfg.Notify.NSQ["1"] = target.NSQArgs{}
-
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
-	return nil
-}
-
-func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "32")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV33{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
+	case "31":
+		// V31 -> V32
+		cfg.Notify.NSQ = make(map[string]target.NSQArgs)
+		cfg.Notify.NSQ["1"] = target.NSQArgs{}
 	}
 
+	// Move to latest.
 	cfg.Version = "33"
 
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
-	return nil
-}
-
-func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
-	bootstrapTrace("migrate config to KV style")
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	// Check if the config version is latest, if not migrate.
-	ok, data, err := checkConfigVersion(objAPI, configFile, "33")
-	if err != nil {
-		return err
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV33{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
 	newCfg := newServerConfig()
 
 	config.SetRegion(newCfg, cfg.Region)
@@ -2797,11 +2608,5 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 		notify.SetNotifyWebhook(newCfg, k, args)
 	}
 
-	if err = saveServerConfig(GlobalContext, objAPI, newCfg); err != nil {
-		return err
-	}
-
-	logger.Info("Configuration file %s migrated from version '%s' to new KV format successfully.",
-		configFile, "33")
-	return nil
+	return newCfg, nil
 }

cmd/config-migrate_test.go

@@ -196,11 +196,12 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	if err := migrateMinioSysConfig(objLayer); err != nil {
+	srvCfg, err := readConfigWithoutMigrate(context.Background(), objLayer)
+	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	if err := migrateMinioSysConfigToKV(objLayer); err != nil {
+	if err = saveServerConfig(GlobalContext, objLayer, srvCfg); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 

cmd/config-versions.go

@@ -717,107 +717,6 @@ type serverConfigV28 struct {
 	Logger logger.Config `json:"logger"`
 }
 
-// serverConfigV29 is just like version '28'.
-type serverConfigV29 serverConfigV28
-
-// serverConfigV30 is just like version '29', stores additionally
-// extensions and mimetypes fields for compression.
-type serverConfigV30 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-
-	// Compression configuration
-	Compression compress.Config `json:"compress"`
-}
-
-// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
-type serverConfigV31 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-
-	// Compression configuration
-	Compression compress.Config `json:"compress"`
-
-	// OpenID configuration
-	OpenID openid.Config `json:"openid"`
-
-	// External policy enforcements.
-	Policy struct {
-		// OPA configuration.
-		OPA opa.Args `json:"opa"`
-
-		// Add new external policy enforcements here.
-	} `json:"policy"`
-}
-
-// serverConfigV32 is just like version '31' with added nsq notifer.
-type serverConfigV32 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notify.Config `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-
-	// Compression configuration
-	Compression compress.Config `json:"compress"`
-
-	// OpenID configuration
-	OpenID openid.Config `json:"openid"`
-
-	// External policy enforcements.
-	Policy struct {
-		// OPA configuration.
-		OPA opa.Args `json:"opa"`
-
-		// Add new external policy enforcements here.
-	} `json:"policy"`
-}
-
 // serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces

cmd/config.go

@@ -25,7 +25,6 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go/v2"
@@ -65,16 +64,16 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 			if withData {
 				data, err := readConfig(ctx, objAPI, obj.Name)
 				if err != nil {
-					return nil, err
+					// ignore history file if not readable.
+					continue
 				}
-				if GlobalKMS != nil {
-					data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-						obj.Bucket: path.Join(obj.Bucket, obj.Name),
-					})
-					if err != nil {
-						return nil, err
-					}
+
+				data, err = decryptData(data, obj.Name)
+				if err != nil {
+					// ignore history file that cannot be loaded.
+					continue
 				}
+
 				cfgEntry.Data = string(data)
 			}
 			configHistory = append(configHistory, cfgEntry)
@@ -110,12 +109,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
 		return nil, err
 	}
 
-	if GlobalKMS != nil {
-		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-			minioMetaBucket: path.Join(minioMetaBucket, historyFile),
-		})
-	}
-	return data, err
+	return decryptData(data, historyFile)
 }
 
 func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@@ -167,14 +161,10 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con
 			return nil, err
 		}
 
-		if GlobalKMS != nil && !utf8.Valid(data) {
-			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: path.Join(minioMetaBucket, configFile),
-			})
-			if err != nil {
-				lookupConfigs(srvCfg, objAPI)
-				return nil, err
-			}
+		data, err = decryptData(data, configFile)
+		if err != nil {
+			lookupConfigs(srvCfg, objAPI)
+			return nil, err
 		}
 	}
 
@@ -205,47 +195,32 @@ func NewConfigSys() *ConfigSys {
 }
 
 // Initialize and load config from remote etcd or local config directory
-func initConfig(objAPI ObjectLayer) error {
+func initConfig(objAPI ObjectLayer) (err error) {
 	bootstrapTrace("load the configuration")
+	defer func() {
+		if err != nil {
+			bootstrapTrace(fmt.Sprintf("loading configuration failed: %v", err))
+		}
+	}()
 
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
 
-	if isFile(getConfigFile()) {
-		if err := migrateConfig(); err != nil {
-			return err
-		}
-	}
-
-	// Check if the config version is latest (kvs), if not migrate.
-	ok, data, err := checkConfigVersion(objAPI, path.Join(minioConfigPrefix, minioConfigFile), "kvs")
-	if err != nil && !errors.Is(err, errConfigNotFound) {
+	srvCfg, err := readConfigWithoutMigrate(GlobalContext, objAPI)
+	if err != nil {
 		return err
 	}
-	if ok {
-		return loadConfig(objAPI, data)
-	}
 
-	// Migrates ${HOME}/.minio/config.json or config.json.deprecated
-	// to '<export_path>/.minio.sys/config/config.json'
-	// ignore if the file doesn't exist.
-	// If etcd is set then migrates /config/config.json
-	// to '<export_path>/.minio.sys/config/config.json'
-	if err := migrateConfigToMinioSys(objAPI); err != nil {
-		return fmt.Errorf("migrateConfigToMinioSys: %w", err)
-	}
+	bootstrapTrace("lookup the configuration")
 
-	// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
-	if err := migrateMinioSysConfig(objAPI); err != nil {
-		return fmt.Errorf("migrateMinioSysConfig: %w", err)
-	}
+	// Override any values from ENVs.
+	lookupConfigs(srvCfg, objAPI)
 
-	// Migrates backend '<export_path>/.minio.sys/config/config.json' to
-	// latest config format.
-	if err := migrateMinioSysConfigToKV(objAPI); err != nil {
-		return fmt.Errorf("migrateMinioSysConfigToKV: %w", err)
-	}
+	// hold the mutex lock before a new config is assigned.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
 
-	return loadConfig(objAPI, nil)
+	return nil
 }

cmd/data-scanner.go

@@ -945,9 +945,9 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		return applyLifecycleAction(lcEvt.Action, oi, ""), 0
+		return applyLifecycleAction(lcEvt, oi), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
+		return applyLifecycleAction(lcEvt, oi), size
 	default:
 		// No action.
 		return false, size
@@ -983,7 +983,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 		InclFreeVersions: true,
 	})
 	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, traceFn)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, nil, traceFn)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -1011,7 +1011,8 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		return objectInfos, nil
 	}
 
-	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	event := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	lim := event.NewerNoncurrentVersions
 	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
 		for _, fi := range fivs {
 			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
@@ -1040,7 +1041,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		}
 
 		// NoncurrentDays not passed yet.
-		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, days)) {
+		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, event.NoncurrentDays)) {
 			// add this version back to remaining versions for
 			// subsequent lifecycle policy applications
 			fivs = append(fivs, fi)
@@ -1055,7 +1056,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		})
 	}
 
-	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
+	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
 	return objectInfos, nil
 }
 
@@ -1153,20 +1154,16 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 	return event
 }
 
-func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
+func applyTransitionRule(event lifecycle.Event, obj ObjectInfo) bool {
 	if obj.DeleteMarker {
 		return false
 	}
-	globalTransitionState.queueTransitionTask(obj, storageClass)
+	globalTransitionState.queueTransitionTask(obj, event)
 	return true
 }
 
-func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
-	action := expireObj
-	if restoredObject {
-		action = expireRestoredObj
-	}
-	if err := expireTransitionedObject(ctx, objLayer, &obj, obj.ToLifecycleOpts(), action); err != nil {
+func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event) bool {
+	if err := expireTransitionedObject(ctx, objLayer, &obj, obj.ToLifecycleOpts(), lcEvent); err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
@@ -1177,13 +1174,13 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 	return true
 }
 
-func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
+func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event) bool {
 	traceFn := globalLifecycleSys.trace(obj)
 	opts := ObjectOptions{
 		Expiration: ExpirationOptions{Expire: true},
 	}
 
-	if applyOnVersion {
+	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = obj.VersionID
 	}
 	if opts.VersionID == "" {
@@ -1201,8 +1198,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		return false
 	}
 
+	tags := auditLifecycleTags(lcEvent)
 	// Send audit for the lifecycle delete operation
-	auditLogLifecycle(ctx, obj, ILMExpiry, traceFn)
+	auditLogLifecycle(ctx, obj, ILMExpiry, tags, traceFn)
 
 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
@@ -1222,20 +1220,19 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 }
 
 // Apply object, object version, restored object or restored object version action on the given object
-func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
-	globalExpiryState.enqueueByDays(obj, restoredObject, applyOnVersion)
+func applyExpiryRule(event lifecycle.Event, obj ObjectInfo) bool {
+	globalExpiryState.enqueueByDays(obj, event)
 	return true
 }
 
 // Perform actions (removal or transitioning of objects), return true the action is successfully performed
-func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
-	switch action {
-	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-		success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
+func applyLifecycleAction(event lifecycle.Event, obj ObjectInfo) (success bool) {
+	switch action := event.Action; action {
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
+		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		success = applyExpiryRule(event, obj)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		success = applyTransitionRule(obj, storageClass)
+		success = applyTransitionRule(event, obj)
 	}
 	return
 }
@@ -1445,7 +1442,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn func(event string)) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]interface{}, traceFn func(event string)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1461,6 +1458,7 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn
 		Bucket:    oi.Bucket,
 		Object:    oi.Name,
 		VersionID: oi.VersionID,
+		Tags:      tags,
 	})
 	traceFn(event)
 }

cmd/data-usage.go

@@ -81,7 +81,7 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 
 				for id, usageInfo := range cache.flattenChildrens(*root) {
 					prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
-					// decodeDirObject to avoid any __XL_DIR__ objects
+					// decodeDirObject to avoid any __XLDIR__ objects
 					m[prefix] += uint64(usageInfo.Size)
 				}
 			}

cmd/disk-cache.go

@@ -33,8 +33,8 @@ import (
 	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	xnet "github.com/minio/pkg/net"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/pkg/wildcard"
 )
 

cmd/encryption-v1.go

@@ -1124,7 +1124,8 @@ func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn
 }
 
 // decryptChecksums will attempt to decode checksums and return it/them if set.
-func (o *ObjectInfo) decryptChecksums() map[string]string {
+// if part > 0, and we have the checksum for the part that will be returned.
+func (o *ObjectInfo) decryptChecksums(part int) map[string]string {
 	data := o.Checksum
 	if len(data) == 0 {
 		return nil
@@ -1137,5 +1138,5 @@ func (o *ObjectInfo) decryptChecksums() map[string]string {
 		}
 		data = decrypted
 	}
-	return hash.ReadCheckSums(data)
+	return hash.ReadCheckSums(data, part)
 }

cmd/erasure-common.go

@@ -23,7 +23,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {

cmd/erasure-healing-common.go

@@ -76,10 +76,14 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 	return time.Unix(0, latest).UTC(), maxima
 }
 
-// commonTime returns a maximally occurring time from a list of time.
-func commonTime(modTimes []time.Time) (modTime time.Time) {
-	modTime, _ = commonTimeAndOccurence(modTimes, 0)
-	return modTime
+// commonTime returns a maximally occurring time from a list of time if it
+// occurs >= quorum, else return timeSentinel
+func commonTime(modTimes []time.Time, quorum int) time.Time {
+	if modTime, count := commonTimeAndOccurence(modTimes, 0); count >= quorum {
+		return modTime
+	}
+
+	return timeSentinel
 }
 
 // Beginning of unix time is treated as sentinel value here.
@@ -157,15 +161,15 @@ func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
 // listOnlineDisks - returns
 // - a slice of disks where disk having 'older' xl.meta (or nothing)
 // are set to nil.
-// - latest (in time) of the maximally occurring modTime(s).
-func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
+// - latest (in time) of the maximally occurring modTime(s), which has at least quorum occurrences.
+func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error, quorum int) (onlineDisks []StorageAPI, modTime time.Time) {
 	onlineDisks = make([]StorageAPI, len(disks))
 
 	// List all the file commit ids from parts metadata.
 	modTimes := listObjectModtimes(partsMetadata, errs)
 
 	// Reduce list of UUIDs to a single common value.
-	modTime = commonTime(modTimes)
+	modTime = commonTime(modTimes, quorum)
 
 	// Create a new online disks slice, which have common uuid.
 	for index, t := range modTimes {

cmd/erasure-healing-common_test.go

@@ -52,7 +52,7 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar
 	var latestFileInfo FileInfo
 
 	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
-	modTime := commonTime(modTimes)
+	modTime := commonTime(modTimes, expectedRQuorum)
 
 	if modTime.IsZero() || modTime.Equal(timeSentinel) {
 		return FileInfo{}, errErasureReadQuorum
@@ -82,8 +82,9 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar
 func TestCommonTime(t *testing.T) {
 	// List of test cases for common modTime.
 	testCases := []struct {
-		times []time.Time
-		time  time.Time
+		times  []time.Time
+		time   time.Time
+		quorum int
 	}{
 		{
 			// 1. Tests common times when slice has varying time elements.
@@ -97,6 +98,7 @@ func TestCommonTime(t *testing.T) {
 				time.Unix(0, 1).UTC(),
 			},
 			time.Unix(0, 3).UTC(),
+			3,
 		},
 		{
 			// 2. Tests common time obtained when all elements are equal.
@@ -110,10 +112,11 @@ func TestCommonTime(t *testing.T) {
 				time.Unix(0, 3).UTC(),
 			},
 			time.Unix(0, 3).UTC(),
+			4,
 		},
 		{
-			// 3. Tests common time obtained when elements have a mixture
-			// of sentinel values.
+			// 3. Tests common time obtained when elements have a mixture of
+			// sentinel values and don't have read quorum on any of the values.
 			[]time.Time{
 				time.Unix(0, 3).UTC(),
 				time.Unix(0, 3).UTC(),
@@ -126,7 +129,8 @@ func TestCommonTime(t *testing.T) {
 				timeSentinel,
 				timeSentinel,
 			},
-			time.Unix(0, 3).UTC(),
+			timeSentinel,
+			5,
 		},
 	}
 
@@ -134,7 +138,7 @@ func TestCommonTime(t *testing.T) {
 	// common modtime. Tests fail if modtime does not match.
 	for i, testCase := range testCases {
 		// Obtain a common mod time from modTimes slice.
-		ctime := commonTime(testCase.times)
+		ctime := commonTime(testCase.times, testCase.quorum)
 		if !testCase.time.Equal(ctime) {
 			t.Errorf("Test case %d, expect to pass but failed. Wanted modTime: %s, got modTime: %s\n", i+1, testCase.time, ctime)
 		}
@@ -151,30 +155,34 @@ func TestListOnlineDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	setObjectLayer(obj)
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	type tamperKind int
 	const (
-		noTamper    tamperKind = iota
-		deletePart  tamperKind = iota
-		corruptPart tamperKind = iota
+		noTamper tamperKind = iota
+		deletePart
+		corruptPart
 	)
-	threeNanoSecs := time.Unix(0, 3).UTC()
-	fourNanoSecs := time.Unix(0, 4).UTC()
-	modTimesThreeNone := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel,
-	}
-	modTimesThreeFour := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
+
+	timeSentinel := time.Unix(1, 0).UTC()
+	threeNanoSecs := time.Unix(3, 0).UTC()
+	fourNanoSecs := time.Unix(4, 0).UTC()
+	modTimesThreeNone := make([]time.Time, 16)
+	modTimesThreeFour := make([]time.Time, 16)
+	for i := 0; i < 16; i++ {
+		// Have 13 good xl.meta, 12 for default parity count = 4 (EC:4) and one
+		// to be tampered with.
+		if i > 12 {
+			modTimesThreeFour[i] = fourNanoSecs
+			modTimesThreeNone[i] = timeSentinel
+			continue
+		}
+		modTimesThreeFour[i] = threeNanoSecs
+		modTimesThreeNone[i] = threeNanoSecs
 	}
+
 	testCases := []struct {
 		modTimes       []time.Time
 		expectedTime   time.Time
@@ -183,10 +191,10 @@ func TestListOnlineDisks(t *testing.T) {
 	}{
 		{
 			modTimes:     modTimesThreeFour,
-			expectedTime: fourNanoSecs,
+			expectedTime: threeNanoSecs,
 			errs: []error{
-				nil, nil, nil, nil, nil, nil, nil, nil, nil,
-				nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
 			},
 			_tamperBackend: noTamper,
 		},
@@ -195,13 +203,10 @@ func TestListOnlineDisks(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks can't access xl.meta.
+				errFileNotFound, errDiskAccessDenied, errDiskNotFound,
 			},
 			_tamperBackend: deletePart,
 		},
@@ -210,13 +215,10 @@ func TestListOnlineDisks(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks don't have xl.meta.
+				errDiskNotFound, errFileNotFound, errFileNotFound,
 			},
 			_tamperBackend: corruptPart,
 		},
@@ -296,7 +298,8 @@ func TestListOnlineDisks(t *testing.T) {
 
 			}
 
-			onlineDisks, modTime := listOnlineDisks(erasureDisks, partsMetadata, test.errs)
+			rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
+			onlineDisks, modTime := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum)
 			if !modTime.Equal(test.expectedTime) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
@@ -325,6 +328,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	setObjectLayer(obj)
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
@@ -337,19 +341,20 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	timeSentinel := time.Unix(1, 0).UTC()
 	threeNanoSecs := time.Unix(3, 0).UTC()
 	fourNanoSecs := time.Unix(4, 0).UTC()
-	modTimesThreeNone := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel,
-	}
-	modTimesThreeFour := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
+	modTimesThreeNone := make([]time.Time, 16)
+	modTimesThreeFour := make([]time.Time, 16)
+	for i := 0; i < 16; i++ {
+		// Have 13 good xl.meta, 12 for default parity count = 4 (EC:4) and one
+		// to be tampered with.
+		if i > 12 {
+			modTimesThreeFour[i] = fourNanoSecs
+			modTimesThreeNone[i] = timeSentinel
+			continue
+		}
+		modTimesThreeFour[i] = threeNanoSecs
+		modTimesThreeNone[i] = threeNanoSecs
 	}
+
 	testCases := []struct {
 		modTimes       []time.Time
 		expectedTime   time.Time
@@ -358,10 +363,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	}{
 		{
 			modTimes:     modTimesThreeFour,
-			expectedTime: fourNanoSecs,
+			expectedTime: threeNanoSecs,
 			errs: []error{
-				nil, nil, nil, nil, nil, nil, nil, nil, nil,
-				nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
 			},
 			_tamperBackend: noTamper,
 		},
@@ -370,13 +375,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks can't access xl.meta.
+				errFileNotFound, errDiskAccessDenied, errDiskNotFound,
 			},
 			_tamperBackend: deletePart,
 		},
@@ -385,13 +387,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks don't have xl.meta.
+				errDiskNotFound, errFileNotFound, errFileNotFound,
 			},
 			_tamperBackend: corruptPart,
 		},
@@ -481,7 +480,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				t.Fatalf("Failed to getLatestFileInfo, expected %v, got %v", errErasureReadQuorum, err)
 			}
 
-			onlineDisks, modTime := listOnlineDisks(erasureDisks, partsMetadata, test.errs)
+			rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
+			onlineDisks, modTime := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum)
 			if !modTime.Equal(test.expectedTime) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
@@ -508,6 +508,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	setObjectLayer(obj)
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
@@ -547,7 +548,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to get quorum consistent fileInfo %v", err)
 	}
 
-	erasureDisks, _ = listOnlineDisks(erasureDisks, partsMetadata, errs)
+	erasureDisks, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)
 
 	filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
 		errs, fi, bucket, object, madmin.HealDeepScan)

cmd/erasure-healing.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
@@ -386,7 +386,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 	// List of disks having latest version of the object xl.meta
 	// (by modtime).
-	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
+	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum)
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.

cmd/erasure-healing_test.go

@@ -567,13 +567,13 @@ func TestHealingDanglingObject(t *testing.T) {
 	// Set globalStoragClass.STANDARD to EC:4 for this test
 	saveSC := globalStorageClass
 	defer func() {
-		globalStorageClass = saveSC
+		globalStorageClass.Update(saveSC)
 	}()
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 4,
 		},
-	}
+	})
 
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)

cmd/erasure-metadata-utils.go

@@ -23,10 +23,9 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
-	"strings"
 
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 // figure out the most commonVersions across disk that satisfies
@@ -173,10 +172,8 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 		errFileVersionNotFound,
 		errDiskNotFound,
 		errUnformattedDisk,
-	}
-	if strings.HasPrefix(bucket, minioMetaBucket) {
-		// listing object might be truncated, ignore such errors from logging.
-		ignoredErrs = append(ignoredErrs, io.ErrUnexpectedEOF)
+		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+		io.EOF,              // some times we would read without locks, ignore these errors
 	}
 	errs := g.Wait()
 	for index, err := range errs {

cmd/erasure-metadata.go

@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/sio"
 )
 
@@ -42,13 +42,6 @@ const minIOErasureUpgraded = "x-minio-internal-erasure-upgraded"
 
 const erasureAlgorithm = "rs-vandermonde"
 
-// byObjectPartNumber is a collection satisfying sort.Interface.
-type byObjectPartNumber []ObjectPartInfo
-
-func (t byObjectPartNumber) Len() int           { return len(t) }
-func (t byObjectPartNumber) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
-func (t byObjectPartNumber) Less(i, j int) bool { return t[i].Number < t[j].Number }
-
 // AddChecksumInfo adds a checksum of a part.
 func (e *ErasureInfo) AddChecksumInfo(ckSumInfo ChecksumInfo) {
 	for i, sum := range e.Checksums {
@@ -308,7 +301,7 @@ func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, act
 	fi.Parts = append(fi.Parts, partInfo)
 
 	// Parts in FileInfo should be in sorted order by part number.
-	sort.Sort(byObjectPartNumber(fi.Parts))
+	sort.Slice(fi.Parts, func(i, j int) bool { return fi.Parts[i].Number < fi.Parts[j].Number })
 }
 
 // ObjectToPartOffset - translate offset of an object to offset of its individual part.

cmd/erasure-multipart.go

@@ -37,8 +37,8 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/mimedb"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
@@ -78,24 +78,22 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 		return fi, nil, err
 	}
 
+	quorum := readQuorum
+	if write {
+		quorum = writeQuorum
+	}
 	// List all online disks.
-	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
+	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs, quorum)
 
-	var quorum int
 	if write {
 		reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 		if reducedErr == errErasureWriteQuorum {
 			return fi, nil, reducedErr
 		}
-
-		quorum = writeQuorum
 	} else {
 		if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 			return fi, nil, reducedErr
 		}
-
-		// Pick one from the first valid metadata.
-		quorum = readQuorum
 	}
 
 	// Pick one from the first valid metadata.
@@ -1137,9 +1135,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 	if checksumType.IsSet() {
-		checksumType |= hash.ChecksumMultipart
+		checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
 		cs := hash.NewChecksumFromData(checksumType, checksumCombined)
-		fi.Checksum = cs.AppendTo(nil, len(fi.Parts))
+		fi.Checksum = cs.AppendTo(nil, checksumCombined)
 		if opts.EncryptFn != nil {
 			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
 		}

cmd/erasure-object.go

@@ -42,8 +42,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/mimedb"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/pkg/wildcard"
 	uatomic "go.uber.org/atomic"
 )
@@ -107,7 +107,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	}
 
 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(storageDisks, metaArr, errs)
+	onlineDisks, modTime := listOnlineDisks(storageDisks, metaArr, errs, readQuorum)
 
 	// Pick latest valid metadata.
 	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
@@ -552,6 +552,8 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		errVolumeNotFound,
 		errFileVersionNotFound,
 		errDiskNotFound,
+		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+		io.EOF,              // some times we would read without locks, ignore these errors
 	}
 
 	errs := g.Wait()
@@ -559,13 +561,6 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		if err == nil {
 			continue
 		}
-		if bucket == minioMetaBucket {
-			// minioMetaBucket "reads" for .metacache are not written with O_SYNC
-			// so there is a potential for them to not fully committed to stable
-			// storage leading to unexpected EOFs. Allow these failures to
-			// be ignored since the caller already ignores them in streamMetadataParts()
-			ignoredErrs = append(ignoredErrs, io.ErrUnexpectedEOF, io.EOF)
-		}
 		if !IsErr(err, ignoredErrs...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], bucket, object, err),
@@ -660,7 +655,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs, readQuorum)
 
 	// Pick latest valid metadata.
 	fi, err = pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
@@ -1091,7 +1086,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.DataDir = mustGetUUID()
-	fi.Checksum = opts.WantChecksum.AppendTo(nil, 0)
+	fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
 	if opts.EncryptFn != nil {
 		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
 	}
@@ -1770,7 +1765,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	}
 
 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs, readQuorum)
 
 	// Pick latest valid metadata.
 	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
@@ -1843,7 +1838,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	}
 
 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs, readQuorum)
 
 	// Pick latest valid metadata.
 	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
@@ -2009,7 +2004,8 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		UserAgent:  "Internal: [ILM-Transition]",
 		Host:       globalLocalNodeName,
 	})
-	auditLogLifecycle(ctx, objInfo, ILMTransition, traceFn)
+	tags := auditLifecycleTags(opts.LifecycleEvent)
+	auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn)
 	return err
 }
 

cmd/erasure-object_test.go

@@ -895,14 +895,14 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 
 	// Object for test case 1 - No StorageClass defined, no MetaData in PutObject
 	object1 := "object1"
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
 		Standard: storageclass.StorageClass{
 			Parity: 4,
 		},
-	}
+	})
 	_, err = obj.PutObject(ctx, bucket, object1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
@@ -939,11 +939,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object4 := "object4"
 	metadata4 := make(map[string]string)
 	metadata4["x-amz-storage-class"] = storageclass.STANDARD
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 6,
 		},
-	}
+	})
 
 	_, err = obj.PutObject(ctx, bucket, object4, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata4})
 	if err != nil {
@@ -962,11 +962,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object5 := "object5"
 	metadata5 := make(map[string]string)
 	metadata5["x-amz-storage-class"] = storageclass.RRS
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
-	}
+	})
 
 	_, err = obj.PutObject(ctx, bucket, object5, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata5})
 	if err != nil {
@@ -980,14 +980,14 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object6 := "object6"
 	metadata6 := make(map[string]string)
 	metadata6["x-amz-storage-class"] = storageclass.STANDARD
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 4,
 		},
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
-	}
+	})
 
 	_, err = obj.PutObject(ctx, bucket, object6, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata6})
 	if err != nil {
@@ -1006,11 +1006,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object7 := "object7"
 	metadata7 := make(map[string]string)
 	metadata7["x-amz-storage-class"] = storageclass.STANDARD
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,
 		},
-	}
+	})
 
 	_, err = obj.PutObject(ctx, bucket, object7, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata7})
 	if err != nil {
@@ -1043,7 +1043,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	for _, tt := range tests {
 		tt := tt
 		t.(*testing.T).Run("", func(t *testing.T) {
-			globalStorageClass = tt.storageClassCfg
+			globalStorageClass.Update(tt.storageClassCfg)
 			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, storageclass.DefaultParityBlocks(len(erasureDisks)))
 			if tt.expectedError != nil && err == nil {
 				t.Errorf("Expected %s, got %s", tt.expectedError, err)

cmd/erasure-server-pool-decom.go

@@ -738,10 +738,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
 			switch {
 			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
-				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt.Action.DeleteRestored())
+				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt)
 				return false
 			case evt.Action.Delete():
-				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+				globalExpiryState.enqueueByDays(objInfo, evt)
 				return true
 			default:
 				return false

cmd/erasure-server-pool-rebalance.go

@@ -465,7 +465,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 
 			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
 			if evt.Action.Delete() {
-				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+				globalExpiryState.enqueueByDays(objInfo, evt)
 				return true
 			}
 

cmd/erasure-server-pool.go

@@ -39,7 +39,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/pkg/wildcard"
 )
 
@@ -1307,7 +1307,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	opts.setBucketMeta(ctx)
 
-	if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
+	if len(prefix) > 0 && maxKeys == 1 && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
 		// - Actifio, Splunk etc.
@@ -1320,7 +1320,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 			if opts.Lifecycle != nil {
 				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
 				if evt.Action.Delete() {
-					globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+					globalExpiryState.enqueueByDays(objInfo, evt)
 					if !evt.Action.DeleteRestored() {
 						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
 						return loi, nil
@@ -2115,6 +2115,7 @@ type HealthResult struct {
 	HealingDrives int
 	PoolID, SetID int
 	WriteQuorum   int
+	UsingDefaults bool
 }
 
 // ReadHealth returns if the cluster can serve read requests
@@ -2209,6 +2210,11 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		}
 	}
 
+	var usingDefaults bool
+	if globalStorageClass.GetParityForSC(storageclass.STANDARD) < 0 {
+		usingDefaults = true
+	}
+
 	for poolIdx := range erasureSetUpCount {
 		for setIdx := range erasureSetUpCount[poolIdx] {
 			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
@@ -2221,6 +2227,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 					PoolID:        poolIdx,
 					SetID:         setIdx,
 					WriteQuorum:   poolWriteQuorums[poolIdx],
+					UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
 				}
 			}
 		}
@@ -2240,8 +2247,9 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	// to look at the healing side of the code.
 	if !opts.Maintenance {
 		return HealthResult{
-			Healthy:     true,
-			WriteQuorum: maximumWriteQuorum,
+			Healthy:       true,
+			WriteQuorum:   maximumWriteQuorum,
+			UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
 		}
 	}
 
@@ -2249,6 +2257,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		Healthy:       len(aggHealStateResult.HealDisks) == 0,
 		HealingDrives: len(aggHealStateResult.HealDisks),
 		WriteQuorum:   maximumWriteQuorum,
+		UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
 	}
 }
 

cmd/erasure-sets.go

@@ -39,8 +39,8 @@ import (
 	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 // setsDsyncLockers is encapsulated type for Close()

cmd/erasure.go

@@ -32,7 +32,7 @@ import (
 	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 // list all errors that can be ignore in a bucket operation.
@@ -94,15 +94,6 @@ func (er erasureObjects) defaultWQuorum() int {
 	return dataCount
 }
 
-// byDiskTotal is a collection satisfying sort.Interface.
-type byDiskTotal []madmin.Disk
-
-func (d byDiskTotal) Len() int      { return len(d) }
-func (d byDiskTotal) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
-func (d byDiskTotal) Less(i, j int) bool {
-	return d[i].TotalSpace < d[j].TotalSpace
-}
-
 func diskErrToDriveState(err error) (state string) {
 	switch {
 	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
@@ -245,7 +236,9 @@ func getStorageInfo(disks []StorageAPI, endpoints []Endpoint) StorageInfo {
 	disksInfo := getDisksInfo(disks, endpoints)
 
 	// Sort so that the first element is the smallest.
-	sort.Sort(byDiskTotal(disksInfo))
+	sort.Slice(disksInfo, func(i, j int) bool {
+		return disksInfo[i].TotalSpace < disksInfo[j].TotalSpace
+	})
 
 	storageInfo := StorageInfo{
 		Disks: disksInfo,

cmd/event-notification.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/url"
 	"strings"
@@ -40,7 +39,6 @@ type EventNotifier struct {
 	targetResCh                chan event.TargetIDResult
 	bucketRulesMap             map[string]event.RulesMap
 	bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
-	eventsQueue                chan eventArgs
 }
 
 // NewEventNotifier - creates new event notification object.
@@ -51,7 +49,6 @@ func NewEventNotifier() *EventNotifier {
 		targetResCh:                make(chan event.TargetIDResult),
 		bucketRulesMap:             make(map[string]event.RulesMap),
 		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
-		eventsQueue:                make(chan eventArgs, 10000),
 	}
 }
 
@@ -105,12 +102,6 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 		return err
 	}
 
-	go func() {
-		for e := range evnot.eventsQueue {
-			evnot.send(e)
-		}
-	}()
-
 	go func() {
 		for res := range evnot.targetResCh {
 			if res.Err != nil {
@@ -210,16 +201,6 @@ func (evnot *EventNotifier) RemoveAllRemoteTargets() {
 
 // Send - sends the event to all registered notification targets
 func (evnot *EventNotifier) Send(args eventArgs) {
-	select {
-	case evnot.eventsQueue <- args:
-	default:
-		// A new goroutine is created for each notification job, eventsQueue is
-		// drained quickly and is not expected to be filled with any scenario.
-		logger.LogIf(context.Background(), errors.New("internal events queue unexpectedly full"))
-	}
-}
-
-func (evnot *EventNotifier) send(args eventArgs) {
 	evnot.RLock()
 	targetIDSet := evnot.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
 	evnot.RUnlock()

cmd/format-erasure.go

@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 const (

cmd/generic-handlers.go

@@ -576,6 +576,8 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			for k := range w.Header() {
 				w.Header().Del(k)
 			}
+			ctx := newContext(r, w, "SiteReplicationUploadForwarding")
+			defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 			globalForwarder.ServeHTTP(w, r)
 			return
 		}

cmd/globals.go

@@ -197,7 +197,7 @@ var (
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
 	// healthcheck readiness deadlines and cors settings.
-	globalAPIConfig = apiConfig{listQuorum: "strict"}
+	globalAPIConfig = apiConfig{listQuorum: "strict", rootAccess: true}
 
 	globalStorageClass storageclass.Config
 
@@ -212,6 +212,7 @@ var (
 	globalTLSCerts *certs.Manager
 
 	globalHTTPServer        *xhttp.Server
+	globalTCPOptions        xhttp.TCPOptions
 	globalHTTPServerErrorCh = make(chan error)
 	globalOSSignalCh        = make(chan os.Signal, 1)
 
@@ -254,6 +255,9 @@ var (
 
 	globalActiveCred auth.Credentials
 
+	// Captures if root credentials are set via ENV.
+	globalCredViaEnv bool
+
 	globalPublicCerts []*x509.Certificate
 
 	globalDomainNames []string      // Root domains for virtual host style requests

cmd/handler-api.go

@@ -51,6 +51,7 @@ type apiConfig struct {
 	deleteCleanupInterval       time.Duration
 	disableODirect              bool
 	gzipObjects                 bool
+	rootAccess                  bool
 }
 
 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -152,6 +153,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
 	t.disableODirect = cfg.DisableODirect
 	t.gzipObjects = cfg.GzipObjects
+	t.rootAccess = cfg.RootAccess
 }
 
 func (t *apiConfig) isDisableODirect() bool {
@@ -168,6 +170,13 @@ func (t *apiConfig) shouldGzipObjects() bool {
 	return t.gzipObjects
 }
 
+func (t *apiConfig) permitRootAccess() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.rootAccess
+}
+
 func (t *apiConfig) getListQuorum() string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()

cmd/healthcheck-handler.go

@@ -28,7 +28,7 @@ import (
 
 const unavailable = "offline"
 
-func isServerInitialized() bool {
+func isServerNotInitialized() bool {
 	return newObjectLayerFn() == nil
 }
 
@@ -36,7 +36,7 @@ func isServerInitialized() bool {
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
-	if isServerInitialized() {
+	if isServerNotInitialized() {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
@@ -55,6 +55,8 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	if result.WriteQuorum > 0 {
 		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))
 	}
+	w.Header().Set(xhttp.MinIOStorageClassDefaults, strconv.FormatBool(result.UsingDefaults))
+
 	if !result.Healthy {
 		// return how many drives are being healed if any
 		if result.HealingDrives > 0 {
@@ -77,7 +79,7 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
-	if isServerInitialized() {
+	if isServerNotInitialized() {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
@@ -104,7 +106,13 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	if isServerInitialized() {
+	peerCall := r.Header.Get("x-minio-from-peer") != ""
+
+	if peerCall {
+		return
+	}
+
+	if isServerNotInitialized() {
 		// Service not initialized yet
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
@@ -121,7 +129,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Verify if KMS is reachable if its configured
-	if GlobalKMS != nil {
+	if GlobalKMS != nil && !peerCall {
 		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
 		defer cancel()
 

cmd/iam-etcd-store.go

@@ -25,7 +25,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -115,26 +114,6 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, it
 	return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
 }
 
-func decryptData(data []byte, itemPath string) ([]byte, error) {
-	var err error
-	if !utf8.Valid(data) && GlobalKMS != nil {
-		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-			minioMetaBucket: path.Join(minioMetaBucket, itemPath),
-		})
-		if err != nil {
-			// This fallback is needed because of a bug, in kms.Context{}
-			// construction during migration.
-			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: itemPath,
-			})
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-	return data, nil
-}
-
 func getIAMConfig(item interface{}, data []byte, itemPath string) error {
 	data, err := decryptData(data, itemPath)
 	if err != nil {

cmd/iam-object-store.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -27,6 +28,7 @@ import (
 	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -91,18 +93,40 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{}
 	return saveConfig(ctx, iamOS.objAPI, objPath, data)
 }
 
+func decryptData(data []byte, objPath string) ([]byte, error) {
+	if utf8.Valid(data) {
+		return data, nil
+	}
+
+	pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+	if err == nil {
+		return pdata, nil
+	}
+	if GlobalKMS != nil {
+		pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+			minioMetaBucket: path.Join(minioMetaBucket, objPath),
+		})
+		if err == nil {
+			return pdata, nil
+		}
+		pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+			minioMetaBucket: objPath,
+		})
+		if err == nil {
+			return pdata, nil
+		}
+	}
+	return nil, err
+}
+
 func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) {
 	data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath, ObjectOptions{})
 	if err != nil {
 		return nil, meta, err
 	}
-	if !utf8.Valid(data) && GlobalKMS != nil {
-		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-			minioMetaBucket: path.Join(minioMetaBucket, objPath),
-		})
-		if err != nil {
-			return nil, meta, err
-		}
+	data, err = decryptData(data, objPath)
+	if err != nil {
+		return nil, meta, err
 	}
 	return data, meta, nil
 }

cmd/iam.go

@@ -286,21 +286,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	// Migrate storage format if needed.
 	for {
-		if etcdClient != nil {
-			// ****  WARNING ****
-			// Migrating to encrypted backend on etcd should happen before initialization of
-			// IAM sub-system, make sure that we do not move the above codeblock elsewhere.
-			if err := migrateIAMConfigsEtcdToEncrypted(retryCtx, etcdClient); err != nil {
-				if errors.Is(err, errEtcdUnreachable) {
-					logger.Info("Connection to etcd timed out. Retrying..")
-					continue
-				}
-				logger.LogIf(ctx, fmt.Errorf("Unable to decrypt an encrypted ETCD backend for IAM users and policies: %w", err))
-				logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
-				return
-			}
-		}
-
 		// Migrate IAM configuration, if necessary.
 		if err := saveIAMFormat(retryCtx, sys.store); err != nil {
 			if configRetriableErrors(err) {
@@ -929,11 +914,12 @@ func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string
 }
 
 type newServiceAccountOpts struct {
-	sessionPolicy *iampolicy.Policy
-	accessKey     string
-	secretKey     string
-	comment       string
-	expiration    *time.Time
+	sessionPolicy              *iampolicy.Policy
+	accessKey                  string
+	secretKey                  string
+	comment                    string
+	expiration                 *time.Time
+	allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
 
 	claims map[string]interface{}
 }
@@ -968,7 +954,9 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	if parentUser == opts.accessKey {
 		return auth.Credentials{}, time.Time{}, errIAMActionNotAllowed
 	}
-
+	if siteReplicatorSvcAcc == opts.accessKey && !opts.allowSiteReplicatorAccount {
+		return auth.Credentials{}, time.Time{}, errIAMActionNotAllowed
+	}
 	m := make(map[string]interface{})
 	m[parentClaim] = parentUser
 

cmd/jwt.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"net/http"
 	"time"
@@ -40,48 +39,15 @@ const (
 
 	// Inter-node JWT token expiry is 15 minutes.
 	defaultInterNodeJWTExpiry = 15 * time.Minute
-
-	// URL JWT token expiry is one minute (might be exposed).
-	defaultURLJWTExpiry = time.Minute
 )
 
 var (
 	errInvalidAccessKeyID = errors.New("The access key ID you provided does not exist in our records")
+	errAccessKeyDisabled  = errors.New("The access key you provided is disabled")
 	errAuthentication     = errors.New("Authentication failed, check your access credentials")
 	errNoAuthToken        = errors.New("JWT token missing")
 )
 
-func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (string, error) {
-	passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
-	if err != nil {
-		return "", err
-	}
-	expiresAt := UTCNow().Add(expiry)
-	return authenticateJWTUsersWithCredentials(passedCredential, expiresAt)
-}
-
-func authenticateJWTUsersWithCredentials(credentials auth.Credentials, expiresAt time.Time) (string, error) {
-	serverCred := globalActiveCred
-	if serverCred.AccessKey != credentials.AccessKey {
-		u, ok := globalIAMSys.GetUser(context.TODO(), credentials.AccessKey)
-		if !ok {
-			return "", errInvalidAccessKeyID
-		}
-		serverCred = u.Credentials
-	}
-
-	if !serverCred.Equal(credentials) {
-		return "", errAuthentication
-	}
-
-	claims := xjwt.NewMapClaims()
-	claims.SetExpiry(expiresAt)
-	claims.SetAccessKey(credentials.AccessKey)
-
-	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
-	return jwt.SignedString([]byte(serverCred.SecretKey))
-}
-
 // cachedAuthenticateNode will cache authenticateNode results for given values up to ttl.
 func cachedAuthenticateNode(ttl time.Duration) func(accessKey, secretKey, audience string) (string, error) {
 	type key struct {
@@ -121,14 +87,6 @@ func authenticateNode(accessKey, secretKey, audience string) (string, error) {
 	return jwt.SignedString([]byte(secretKey))
 }
 
-func authenticateWeb(accessKey, secretKey string) (string, error) {
-	return authenticateJWTUsers(accessKey, secretKey, defaultJWTExpiry)
-}
-
-func authenticateURL(accessKey, secretKey string) (string, error) {
-	return authenticateJWTUsers(accessKey, secretKey, defaultURLJWTExpiry)
-}
-
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
@@ -142,15 +100,24 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 	}
 	claims := xjwt.NewMapClaims()
 	if err := xjwt.ParseWithClaims(token, claims, func(claims *xjwt.MapClaims) ([]byte, error) {
-		if claims.AccessKey == globalActiveCred.AccessKey {
-			return []byte(globalActiveCred.SecretKey), nil
+		if claims.AccessKey != globalActiveCred.AccessKey {
+			u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
+			if !ok {
+				// Credentials will be invalid but for disabled
+				// return a different error in such a scenario.
+				if u.Credentials.Status == auth.AccountOff {
+					return nil, errAccessKeyDisabled
+				}
+				return nil, errInvalidAccessKeyID
+			}
+			cred := u.Credentials
+			return []byte(cred.SecretKey), nil
+		} // this means claims.AccessKey == rootAccessKey
+		if !globalAPIConfig.permitRootAccess() {
+			// if root access is disabled, fail this request.
+			return nil, errAccessKeyDisabled
 		}
-		u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
-		if !ok {
-			return nil, errInvalidAccessKeyID
-		}
-		cred := u.Credentials
-		return []byte(cred.SecretKey), nil
+		return []byte(globalActiveCred.SecretKey), nil
 	}); err != nil {
 		return claims, nil, false, errAuthentication
 	}
@@ -173,6 +140,11 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 			claims.MapClaims[k] = v
 		}
 
+		// if root access is disabled, disable all its service accounts and temporary credentials.
+		if ucred.ParentUser == globalActiveCred.AccessKey && !globalAPIConfig.permitRootAccess() {
+			return nil, nil, false, errAccessKeyDisabled
+		}
+
 		// Now check if we have a sessionPolicy.
 		if _, ok = eclaims[iampolicy.SessionPolicyName]; ok {
 			owner = false

cmd/jwt_test.go

@@ -25,78 +25,9 @@ import (
 	"time"
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
-	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
 )
 
-func testAuthenticate(authType string, t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	obj, fsDir, err := prepareFS(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-	if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
-		t.Fatal(err)
-	}
-
-	cred, err := auth.GetNewCredentials()
-	if err != nil {
-		t.Fatalf("Error getting new credentials: %s", err)
-	}
-
-	globalActiveCred = cred
-
-	// Define test cases.
-	testCases := []struct {
-		accessKey   string
-		secretKey   string
-		expectedErr error
-	}{
-		// Access key (less than 3 chrs) too small.
-		{"u1", cred.SecretKey, auth.ErrInvalidAccessKeyLength},
-		// Secret key (less than 8 chrs) too small.
-		{cred.AccessKey, "pass", auth.ErrInvalidSecretKeyLength},
-		// Authentication error.
-		{"myuser", "mypassword", errInvalidAccessKeyID},
-		// Authentication error.
-		{cred.AccessKey, "mypassword", errAuthentication},
-		// Success.
-		{cred.AccessKey, cred.SecretKey, nil},
-	}
-
-	// Run tests.
-	for _, testCase := range testCases {
-		var err error
-		if authType == "web" {
-			_, err = authenticateWeb(testCase.accessKey, testCase.secretKey)
-		} else if authType == "url" {
-			_, err = authenticateURL(testCase.accessKey, testCase.secretKey)
-		}
-
-		if testCase.expectedErr != nil {
-			if err == nil {
-				t.Fatalf("%+v: expected: %s, got: <nil>", testCase, testCase.expectedErr)
-			}
-			if testCase.expectedErr.Error() != err.Error() {
-				t.Fatalf("%+v: expected: %s, got: %s", testCase, testCase.expectedErr, err)
-			}
-		} else if err != nil {
-			t.Fatalf("%+v: expected: <nil>, got: %s", testCase, err)
-		}
-	}
-}
-
-func TestAuthenticateWeb(t *testing.T) {
-	testAuthenticate("web", t)
-}
-
-func TestAuthenticateURL(t *testing.T) {
-	testAuthenticate("url", t)
-}
-
 func getTokenString(accessKey, secretKey string) (string, error) {
 	claims := xjwt.NewMapClaims()
 	claims.SetExpiry(UTCNow().Add(defaultJWTExpiry))
@@ -258,24 +189,3 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		}
 	})
 }
-
-func BenchmarkAuthenticateWeb(b *testing.B) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	obj, fsDir, err := prepareFS(ctx)
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-	if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
-		b.Fatal(err)
-	}
-
-	creds := globalActiveCred
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		authenticateWeb(creds.AccessKey, creds.SecretKey)
-	}
-}

cmd/metacache-entries.go

@@ -55,7 +55,7 @@ func (e metaCacheEntry) isObject() bool {
 	return len(e.metadata) > 0
 }
 
-// isObjectDir returns if the entry is representing an object__XL_DIR__
+// isObjectDir returns if the entry is representing an object/
 func (e metaCacheEntry) isObjectDir() bool {
 	return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator)
 }

cmd/metacache-server-pool.go

@@ -210,10 +210,6 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			}
 		}()
 		o.ID = ""
-
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Resuming listing from drives failed %w, proceeding to do raw listing", err))
-		}
 	}
 
 	// Do listing in-place.
@@ -399,7 +395,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 			if o.Lifecycle != nil {
 				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
 				if evt.Action.Delete() {
-					globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+					globalExpiryState.enqueueByDays(objInfo, evt)
 					if !evt.Action.DeleteRestored() {
 						continue
 					} // queue version for replication upon expired restored copies if needed.

cmd/metrics-v2.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -114,6 +114,7 @@ const (
 	capacityRawSubsystem      MetricSubsystem = "capacity_raw"
 	capacityUsableSubsystem   MetricSubsystem = "capacity_usable"
 	diskSubsystem             MetricSubsystem = "disk"
+	storageClassSubsystem     MetricSubsystem = "storage_class"
 	fileDescriptorSubsystem   MetricSubsystem = "file_descriptor"
 	goRoutines                MetricSubsystem = "go_routine"
 	ioSubsystem               MetricSubsystem = "io"
@@ -430,6 +431,26 @@ func getNodeDrivesTotalMD() MetricDescription {
 	}
 }
 
+func getNodeStandardParityMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: storageClassSubsystem,
+		Name:      "standard_parity",
+		Help:      "standard storage class parity",
+		Type:      gaugeMetric,
+	}
+}
+
+func getNodeRRSParityMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: storageClassSubsystem,
+		Name:      "rrs_parity",
+		Help:      "reduced redundancy storage class parity",
+		Type:      gaugeMetric,
+	}
+}
+
 func getNodeDrivesFreeInodes() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -936,7 +957,7 @@ func getS3TTFBDistributionMD() MetricDescription {
 		Subsystem: timeSubsystem,
 		Name:      ttfbDistribution,
 		Help:      "Distribution of the time to first byte across API calls",
-		Type:      gaugeMetric,
+		Type:      histogramMetric,
 	}
 }
 
@@ -2187,23 +2208,33 @@ func getLocalStorageMetrics() *MetricsGroup {
 				Value:          float64(disk.FreeInodes),
 				VariableLabels: map[string]string{"disk": disk.DrivePath},
 			})
-
-			metrics = append(metrics, Metric{
-				Description: getNodeDrivesOfflineTotalMD(),
-				Value:       float64(offlineDrives.Sum()),
-			})
-
-			metrics = append(metrics, Metric{
-				Description: getNodeDrivesOnlineTotalMD(),
-				Value:       float64(onlineDrives.Sum()),
-			})
-
-			metrics = append(metrics, Metric{
-				Description: getNodeDrivesTotalMD(),
-				Value:       float64(totalDrives.Sum()),
-			})
-
 		}
+
+		metrics = append(metrics, Metric{
+			Description: getNodeDrivesOfflineTotalMD(),
+			Value:       float64(offlineDrives.Sum()),
+		})
+
+		metrics = append(metrics, Metric{
+			Description: getNodeDrivesOnlineTotalMD(),
+			Value:       float64(onlineDrives.Sum()),
+		})
+
+		metrics = append(metrics, Metric{
+			Description: getNodeDrivesTotalMD(),
+			Value:       float64(totalDrives.Sum()),
+		})
+
+		metrics = append(metrics, Metric{
+			Description: getNodeStandardParityMD(),
+			Value:       float64(storageInfo.Backend.StandardSCParity),
+		})
+
+		metrics = append(metrics, Metric{
+			Description: getNodeRRSParityMD(),
+			Value:       float64(storageInfo.Backend.RRSCParity),
+		})
+
 		return
 	})
 	return mg

cmd/net.go

@@ -111,25 +111,6 @@ func getHostIP(host string) (ipList set.StringSet, err error) {
 	return ipList, err
 }
 
-// byLastOctetValue implements sort.Interface used in sorting a list
-// of ip address by their last octet value in descending order.
-type byLastOctetValue []net.IP
-
-func (n byLastOctetValue) Len() int      { return len(n) }
-func (n byLastOctetValue) Swap(i, j int) { n[i], n[j] = n[j], n[i] }
-func (n byLastOctetValue) Less(i, j int) bool {
-	// This case is needed when all ips in the list
-	// have same last octets, Following just ensures that
-	// 127.0.0.1 is moved to the end of the list.
-	if n[i].IsLoopback() {
-		return false
-	}
-	if n[j].IsLoopback() {
-		return true
-	}
-	return []byte(n[i].To4())[3] > []byte(n[j].To4())[3]
-}
-
 // sortIPs - sort ips based on higher octects.
 // The logic to sort by last octet is implemented to
 // prefer CIDRs with higher octects, this in-turn skips the
@@ -152,7 +133,18 @@ func sortIPs(ipList []string) []string {
 		}
 	}
 
-	sort.Sort(byLastOctetValue(ipV4s))
+	sort.Slice(ipV4s, func(i, j int) bool {
+		// This case is needed when all ips in the list
+		// have same last octets, Following just ensures that
+		// 127.0.0.1 is moved to the end of the list.
+		if ipV4s[i].IsLoopback() {
+			return false
+		}
+		if ipV4s[j].IsLoopback() {
+			return true
+		}
+		return []byte(ipV4s[i].To4())[3] > []byte(ipV4s[j].To4())[3]
+	})
 
 	var ips []string
 	for _, ip := range ipV4s {
@@ -215,19 +207,6 @@ func isHostIP(ipAddress string) bool {
 	return net.ParseIP(host) != nil
 }
 
-// checkPortAvailability - check if given host and port is already in use.
-// Note: The check method tries to listen on given port and closes it.
-// It is possible to have a disconnected client in this tiny window of time.
-func checkPortAvailability(host, port string) (err error) {
-	l, err := net.Listen("tcp", net.JoinHostPort(host, port))
-	if err != nil {
-		return err
-	}
-	// As we are able to listen on this network, the port is not in use.
-	// Close the listener and continue check other networks.
-	return l.Close()
-}
-
 // extractHostPort - extracts host/port from many address formats
 // such as, ":9000", "localhost:9000", "http://localhost:9000/"
 func extractHostPort(hostAddr string) (string, string, error) {

cmd/net_test.go

@@ -20,9 +20,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"net"
 	"reflect"
-	"runtime"
 	"testing"
 
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -180,61 +178,6 @@ func TestGetAPIEndpoints(t *testing.T) {
 	}
 }
 
-// Ask the kernel for a free open port.
-func getFreePort() string {
-	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
-	if err != nil {
-		panic(err)
-	}
-
-	l, err := net.ListenTCP("tcp", addr)
-	if err != nil {
-		panic(err)
-	}
-	defer l.Close()
-	return fmt.Sprintf("%d", l.Addr().(*net.TCPAddr).Port)
-}
-
-// Tests for port availability logic written for server startup sequence.
-func TestCheckPortAvailability(t *testing.T) {
-	// Make a port is not available.
-	port := getFreePort()
-	listener, err := net.Listen("tcp", net.JoinHostPort("", port))
-	if err != nil {
-		t.Fatalf("Unable to listen on port %v", port)
-	}
-	defer listener.Close()
-
-	testCases := []struct {
-		host        string
-		port        string
-		expectedErr error
-	}{
-		{"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
-		{"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)},
-		{"", getFreePort(), nil},
-	}
-
-	for _, testCase := range testCases {
-		// On MS Windows and Mac, skip checking error case due to https://github.com/golang/go/issues/7598
-		if (runtime.GOOS == globalWindowsOSName || runtime.GOOS == globalMacOSName || runtime.GOOS == "solaris") && testCase.expectedErr != nil {
-			continue
-		}
-
-		err := checkPortAvailability(testCase.host, testCase.port)
-		switch {
-		case testCase.expectedErr == nil:
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		case err == nil:
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		case testCase.expectedErr.Error() != err.Error():
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-	}
-}
-
 func TestCheckLocalServerAddr(t *testing.T) {
 	testCases := []struct {
 		serverAddr  string

cmd/notification.go

@@ -32,8 +32,8 @@ import (
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	xnet "github.com/minio/pkg/net"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 // This file contains peer related notifications. For sending notifications to
@@ -883,11 +883,13 @@ func (sys *NotificationSys) GetProcInfo(ctx context.Context) []madmin.ProcInfo {
 	return reply
 }
 
+// Construct a list of offline disks information for a given node.
+// If offlineHost is empty, do it for the local disks.
 func getOfflineDisks(offlineHost string, endpoints EndpointServerPools) []madmin.Disk {
 	var offlineDisks []madmin.Disk
 	for _, pool := range endpoints {
 		for _, ep := range pool.Endpoints {
-			if offlineHost == ep.Host {
+			if offlineHost == "" && ep.IsLocal || offlineHost == ep.Host {
 				offlineDisks = append(offlineDisks, madmin.Disk{
 					Endpoint: ep.String(),
 					State:    string(madmin.ItemOffline),
@@ -947,8 +949,6 @@ func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
 				info.Endpoint = client.host.String()
 				info.State = string(madmin.ItemOffline)
 				info.Disks = getOfflineDisks(info.Endpoint, globalEndpoints)
-			} else {
-				info.State = string(madmin.ItemOnline)
 			}
 			reply[idx] = info
 		}(client, i)

cmd/object-api-datatypes.go

@@ -552,13 +552,6 @@ type CompletePart struct {
 	ChecksumSHA256 string
 }
 
-// CompletedParts - is a collection satisfying sort.Interface.
-type CompletedParts []CompletePart
-
-func (a CompletedParts) Len() int           { return len(a) }
-func (a CompletedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a CompletedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
-
 // CompleteMultipartUpload - represents list of parts which are completed, this is sent by the
 // client during CompleteMultipartUpload request.
 type CompleteMultipartUpload struct {

cmd/object-api-interface.go

@@ -28,6 +28,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/hash"
 
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/replication"
 	xioutil "github.com/minio/minio/internal/ioutil"
 )
@@ -60,6 +61,7 @@ type ObjectOptions struct {
 	DeleteReplication ReplicationState    // Represents internal replication state needed for Delete replication
 	Transition        TransitionOptions
 	Expiration        ExpirationOptions
+	LifecycleEvent    lifecycle.Event
 
 	WantChecksum *hash.Checksum // x-amz-checksum-XXX checksum sent to PutObject/ CompleteMultipartUpload.
 

cmd/object-api-multipart_test.go

@@ -1197,14 +1197,14 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 	objectNames := []string{"minio-object-1.txt"}
 	uploadIDs := []string{}
 
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
 		Standard: storageclass.StorageClass{
 			Parity: 4,
 		},
-	}
+	})
 
 	// bucketnames[0].
 	// objectNames[0].

cmd/object-api-options.go

@@ -174,6 +174,11 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
 	// benefits of replication, make sure to apply version suspension
 	// only at bucket level instead.
 	opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket)
+	// For directory objects, delete `null` version permanently.
+	if isDirObject(object) && opts.VersionID == "" {
+		opts.VersionID = nullVersionID
+	}
+
 	delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
 	if delMarker != "" {
 		switch delMarker {
@@ -321,9 +326,16 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	if err != nil {
 		return opts, err
 	}
+
 	opts.VersionID = vid
 	opts.Versioned = versioned
 	opts.VersionSuspended = versionSuspended
+
+	// For directory objects skip creating new versions.
+	if isDirObject(object) && vid == "" {
+		opts.VersionID = nullVersionID
+	}
+
 	opts.MTime = mtime
 	opts.ReplicationSourceLegalholdTimestamp = lholdtimestmp
 	opts.ReplicationSourceRetentionTimestamp = retaintimestmp

cmd/object-handlers-common.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -340,10 +341,10 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
 			lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 		}
 	}
-	hash.AddChecksumHeader(w, objInfo.decryptChecksums())
+	hash.AddChecksumHeader(w, objInfo.decryptChecksums(0))
 }
 
-func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete) {
+func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete, lcEvent lifecycle.Event) {
 	for remaining := toDel; len(remaining) > 0; toDel = remaining {
 		if len(toDel) > maxDeleteList {
 			remaining = toDel[maxDeleteList:]
@@ -373,11 +374,13 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 				VersionID: dobj.VersionID,
 			}
 			traceFn := globalLifecycleSys.trace(oi)
+			tags := auditLifecycleTags(lcEvent)
+
 			// Send audit for the lifecycle delete operation
 			auditLogLifecycle(
 				ctx,
 				oi,
-				ILMExpiry, traceFn)
+				ILMExpiry, tags, traceFn)
 
 			sendEvent(eventArgs{
 				EventName:  event.ObjectRemovedDelete,

cmd/object-handlers.go

@@ -476,11 +476,11 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			act := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo).Action
-			if act.Delete() {
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
-				applyExpiryRule(objInfo, act.DeleteRestored(), act.DeleteVersioned())
-				if !act.DeleteRestored() {
+				applyExpiryRule(event, objInfo)
+				if !event.Action.DeleteRestored() {
 					// If the ILM action is not on restored object return error.
 					writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
 					return
@@ -513,8 +513,9 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 	}
 
-	if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" {
-		hash.AddChecksumHeader(w, objInfo.decryptChecksums())
+	if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
+		// AWS S3 silently drops checksums on range requests.
+		hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber))
 	}
 
 	if err = setObjectHeaders(w, objInfo, rs, opts); err != nil {
@@ -544,7 +545,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			return
 		}
 		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client %w", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
 		}
 		return
 	}
@@ -555,7 +556,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			return
 		}
 		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client %w", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
 		}
 		return
 	}
@@ -729,11 +730,11 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			act := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo).Action
-			if act.Delete() {
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
-				applyExpiryRule(objInfo, act.DeleteRestored(), act.DeleteVersioned())
-				if !act.DeleteRestored() {
+				applyExpiryRule(event, objInfo)
+				if !event.Action.DeleteRestored() {
 					// If the ILM action is not on restored object return error.
 					writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
 					return
@@ -800,8 +801,9 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 	}
 
-	if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" {
-		hash.AddChecksumHeader(w, objInfo.decryptChecksums())
+	if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil {
+		// AWS S3 silently drops checksums on range requests.
+		hash.AddChecksumHeader(w, objInfo.decryptChecksums(opts.PartNumber))
 	}
 
 	// Set standard object headers.
@@ -2140,6 +2142,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return err
 		}
 		opts.MTime = info.ModTime()
+		if opts.MTime.Unix() <= 0 {
+			opts.MTime = UTCNow()
+		}
 		opts.IndexCB = idxCb
 
 		retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)

cmd/object-multipart-handlers.go

@@ -890,7 +890,9 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
 		return
 	}
-	if !sort.IsSorted(CompletedParts(complMultipartUpload.Parts)) {
+	if !sort.SliceIsSorted(complMultipartUpload.Parts, func(i, j int) bool {
+		return complMultipartUpload.Parts[i].PartNumber < complMultipartUpload.Parts[j].PartNumber
+	}) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartOrder), r.URL)
 		return
 	}

cmd/peer-s3-client.go

@@ -29,8 +29,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
-	"github.com/minio/minio/internal/sync/errgroup"
 	xnet "github.com/minio/pkg/net"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 var errPeerOffline = errors.New("peer is offline")
@@ -273,9 +273,24 @@ func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts Dele
 	errs := g.Wait()
 	errs = append(errs, deleteBucketLocal(ctx, bucket, opts))
 
-	quorum := (len(sys.allPeerClients) / 2) + 1
-	err := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum)
-	return toObjectErr(err, bucket)
+	var errReturn error
+	for _, err := range errs {
+		if errReturn == nil && err != nil {
+			// always return first error
+			errReturn = toObjectErr(err, bucket)
+			break
+		}
+	}
+
+	for _, err := range errs {
+		if err == nil && errReturn != nil {
+			// re-create successful deletes, since we are return an error.
+			sys.MakeBucket(ctx, bucket, MakeBucketOptions{})
+			break
+		}
+	}
+
+	return errReturn
 }
 
 // DeleteBucket deletes bucket on a peer

cmd/peer-s3-server.go

@@ -25,8 +25,8 @@ import (
 	"sort"
 
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/mux"
+	"github.com/minio/pkg/sync/errgroup"
 )
 
 const (

cmd/prepare-storage.go

@@ -125,15 +125,16 @@ func isServerResolvable(endpoint Endpoint, timeout time.Duration) error {
 	}
 
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
+	defer cancel()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL.String(), nil)
 	if err != nil {
-		cancel()
 		return err
 	}
 
+	req.Header.Set("x-minio-from-peer", "true")
+
 	resp, err := httpClient.Do(req)
-	cancel()
 	if err != nil {
 		return err
 	}

cmd/s3-zip-handlers.go

@@ -224,7 +224,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 			return
 		}
 		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client %w", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
 		}
 		return
 	}
@@ -235,7 +235,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 			return
 		}
 		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client %w", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
 		}
 		return
 	}

cmd/server-main.go

@@ -34,6 +34,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/coreos/go-systemd/v22/daemon"
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7"
@@ -43,6 +44,7 @@ import (
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -105,6 +107,19 @@ var ServerFlags = []cli.Flag{
 		Value:  10 * time.Minute,
 		EnvVar: "MINIO_CONN_WRITE_DEADLINE",
 	},
+	cli.DurationFlag{
+		Name:   "conn-user-timeout",
+		Usage:  "custom TCP_USER_TIMEOUT for socket buffers",
+		Hidden: true,
+		Value:  10 * time.Minute,
+		EnvVar: "MINIO_CONN_USER_TIMEOUT",
+	},
+	cli.StringFlag{
+		Name:   "interface",
+		Usage:  "bind to right VRF device for MinIO services",
+		Hidden: true,
+		EnvVar: "MINIO_INTERFACE",
+	},
 	cli.StringSliceFlag{
 		Name:  "ftp",
 		Usage: "enable and configure an FTP(Secure) server",
@@ -252,11 +267,26 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	globalInternodeTransport = NewInternodeHTTPTransport()()
 	globalRemoteTargetTransport = NewRemoteTargetHTTPTransport()()
 
+	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
+		PassHost:     true,
+		RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
+		Logger: func(err error) {
+			if err != nil && !errors.Is(err, context.Canceled) {
+				logger.LogIf(GlobalContext, err)
+			}
+		},
+	})
+
+	globalTCPOptions = xhttp.TCPOptions{
+		UserTimeout: int(ctx.Duration("conn-user-timeout").Milliseconds()),
+		Interface:   ctx.String("interface"),
+	}
+
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
 	// To avoid this error situation we check for port availability.
-	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the server")
+	logger.FatalIf(xhttp.CheckPortAvailability(globalMinioHost, globalMinioPort, globalTCPOptions), "Unable to start the server")
 
 	globalIsErasure = (setupType == ErasureSetupType)
 	globalIsDistErasure = (setupType == DistErasureSetupType)
@@ -382,17 +412,8 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 	// Once the config is fully loaded, initialize the new object layer.
 	setObjectLayer(newObject)
 
-	// ****  WARNING ****
-	// Migrating to encrypted backend should happen before initialization of any
-	// sub-systems, make sure that we do not move the above codeblock elsewhere.
-
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	lockTimeout := newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
-		timeout: 5 * time.Second,
-		minimum: 3 * time.Second,
-	})
-
 	for {
 		select {
 		case <-ctx.Done():
@@ -401,57 +422,23 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 		default:
 		}
 
-		bootstrapTrace("trying to acquire transaction.lock")
-
-		// Make sure to hold lock for entire migration to avoid
-		// such that only one server should migrate the entire config
-		// at a given time, this big transaction lock ensures this
-		// appropriately. This is also true for rotation of encrypted
-		// content.
-		txnLk := newObject.NewNSLock(minioMetaBucket, minioConfigPrefix+"/transaction.lock")
-
-		// let one of the server acquire the lock, if not let them timeout.
-		// which shall be retried again by this loop.
-		lkctx, err := txnLk.GetLock(ctx, lockTimeout)
-		if err != nil {
-			logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock")
-			waitDuration := time.Duration(r.Float64() * 5 * float64(time.Second))
-			bootstrapTrace(fmt.Sprintf("lock not available. error: %v. sleeping for %v before retry", err, waitDuration))
-
-			// Sleep 0 -> 5 seconds, provider a higher range such that sleeps()
-			// and retries for lock are more spread out, needed orchestrated
-			// systems take 30s minimum to respond to DNS resolvers.
-			//
-			// Do not change this value.
-			time.Sleep(waitDuration)
-			continue
-		}
-
 		// These messages only meant primarily for distributed setup, so only log during distributed setup.
 		if globalIsDistErasure {
-			logger.Info("Waiting for all MinIO sub-systems to be initialized.. lock acquired")
+			logger.Info("Waiting for all MinIO sub-systems to be initialize...")
 		}
 
-		// Migrate all backend configs to encrypted backend configs, optionally
-		// handles rotating keys for encryption, if there is any retriable failure
-		// that shall be retried if there is an error.
-		if err = handleEncryptedConfigBackend(newObject); err == nil {
-			// Upon success migrating the config, initialize all sub-systems
-			// if all sub-systems initialized successfully return right away
-			if err = initConfigSubsystem(lkctx.Context(), newObject); err == nil {
-				txnLk.Unlock(lkctx)
-				// All successful return.
-				if globalIsDistErasure {
-					// These messages only meant primarily for distributed setup, so only log during distributed setup.
-					logger.Info("All MinIO sub-systems initialized successfully in %s", time.Since(t1))
-				}
-				return nil
+		// Upon success migrating the config, initialize all sub-systems
+		// if all sub-systems initialized successfully return right away
+		err := initConfigSubsystem(ctx, newObject)
+		if err == nil {
+			// All successful return.
+			if globalIsDistErasure {
+				// These messages only meant primarily for distributed setup, so only log during distributed setup.
+				logger.Info("All MinIO sub-systems initialized successfully in %s", time.Since(t1))
 			}
+			return nil
 		}
 
-		// Unlock the transaction lock and allow other nodes to acquire the lock if possible.
-		txnLk.Unlock(lkctx)
-
 		if configRetriableErrors(err) {
 			logger.Info("Waiting for all MinIO sub-systems to be initialized.. possible cause (%v)", err)
 			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
@@ -601,7 +588,8 @@ func serverMain(ctx *cli.Context) {
 		UseIdleTimeout(ctx.Duration("idle-timeout")).
 		UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
 		UseBaseContext(GlobalContext).
-		UseCustomLogger(log.New(io.Discard, "", 0)) // Turn-off random logging by Go stdlib
+		UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
+		UseTCPOptions(globalTCPOptions)
 
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
@@ -673,7 +661,10 @@ func serverMain(ctx *cli.Context) {
 			setConsoleSrv(srv)
 
 			go func() {
-				logger.FatalIf(newConsoleServerFn().Serve(), "Unable to initialize console server")
+				server := newConsoleServerFn()
+				logger.FatalIf(server.Listen(), "Unable to initialize console server's sockets")
+				daemon.SdNotify(false, daemon.SdNotifyReady)
+				logger.FatalIf(server.Serve(), "Unable to initialize console server")
 			}()
 		}
 
@@ -757,7 +748,7 @@ func serverMain(ctx *cli.Context) {
 		printStartupMessage(getAPIEndpoints(), err)
 
 		// Print a warning at the end of the startup banner so it is more noticeable
-		if globalStorageClass.GetParityForSC("") == 0 {
+		if newObject.BackendInfo().StandardSCParity == 0 {
 			logger.Error("Warning: The standard parity is set to 0. This can lead to data loss.")
 		}
 	}()

cmd/signals.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/coreos/go-systemd/v22/daemon"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -80,17 +81,23 @@ func handleSignals() {
 			exit(stopProcess())
 		case osSignal := <-globalOSSignalCh:
 			logger.Info("Exiting on signal: %s", strings.ToUpper(osSignal.String()))
+			daemon.SdNotify(false, daemon.SdNotifyStopping)
 			exit(stopProcess())
 		case signal := <-globalServiceSignalCh:
 			switch signal {
 			case serviceRestart:
 				logger.Info("Restarting on service signal")
+				daemon.SdNotify(false, daemon.SdNotifyReloading)
 				stop := stopProcess()
 				rerr := restartProcess()
+				if rerr == nil {
+					daemon.SdNotify(false, daemon.SdNotifyReady)
+				}
 				logger.LogIf(context.Background(), rerr)
 				exit(stop && rerr == nil)
 			case serviceStop:
 				logger.Info("Stopping on service signal")
+				daemon.SdNotify(false, daemon.SdNotifyStopping)
 				exit(stopProcess())
 			}
 		}

cmd/signature-v4-utils.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the
@@ -169,7 +170,16 @@ func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, A
 	}
 	cred.Claims = claims
 
-	owner := cred.AccessKey == globalActiveCred.AccessKey
+	owner := cred.AccessKey == globalActiveCred.AccessKey || (cred.ParentUser == globalActiveCred.AccessKey && cred.AccessKey != siteReplicatorSvcAcc)
+	if owner && !globalAPIConfig.permitRootAccess() {
+		// We disable root access and its service accounts if asked for.
+		return cred, owner, ErrAccessKeyDisabled
+	}
+
+	if _, ok := claims[iampolicy.SessionPolicyName]; ok {
+		owner = false
+	}
+
 	return cred, owner, ErrNone
 }
 

cmd/site-replication.go

@@ -459,8 +459,9 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 			return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
 		}
 		svcCred, _, err = globalIAMSys.NewServiceAccount(ctx, sites[selfIdx].AccessKey, nil, newServiceAccountOpts{
-			accessKey: siteReplicatorSvcAcc,
-			secretKey: secretKey,
+			accessKey:                  siteReplicatorSvcAcc,
+			secretKey:                  secretKey,
+			allowSiteReplicatorAccount: true,
 		})
 		if err != nil {
 			return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
@@ -558,8 +559,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 	return result, nil
 }
 
-// PeerJoinReq - internal API handler to respond to a peer cluster's request
-// to join.
+// PeerJoinReq - internal API handler to respond to a peer cluster's request to join.
 func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJoinReq) error {
 	var ourName string
 	for d, p := range arg.Peers {
@@ -575,8 +575,9 @@ func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJ
 	_, _, err := globalIAMSys.GetServiceAccount(ctx, arg.SvcAcctAccessKey)
 	if err == errNoSuchServiceAccount {
 		_, _, err = globalIAMSys.NewServiceAccount(ctx, arg.SvcAcctParent, nil, newServiceAccountOpts{
-			accessKey: arg.SvcAcctAccessKey,
-			secretKey: arg.SvcAcctSecretKey,
+			accessKey:                  arg.SvcAcctAccessKey,
+			secretKey:                  arg.SvcAcctSecretKey,
+			allowSiteReplicatorAccount: arg.SvcAcctAccessKey == siteReplicatorSvcAcc,
 		})
 	}
 	if err != nil {

cmd/storage-rest-client.go

@@ -268,7 +268,7 @@ func (client *storageRESTClient) SetDiskID(id string) {
 }
 
 // DiskInfo - fetch disk information for a remote disk.
-func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
+func (client *storageRESTClient) DiskInfo(_ context.Context) (info DiskInfo, err error) {
 	if !client.IsOnline() {
 		// make sure to check if the disk is offline, since the underlying
 		// value is cached we should attempt to invalidate it if such calls
@@ -281,7 +281,7 @@ func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, e
 		client.diskInfoCache.TTL = time.Second
 		client.diskInfoCache.Update = func() (interface{}, error) {
 			var info DiskInfo
-			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 			defer cancel()
 			respBody, err := client.call(ctx, storageRESTMethodDiskInfo, nil, nil, -1)
 			if err != nil {

cmd/update.go

@@ -403,7 +403,7 @@ func parseReleaseData(data string) (sha256Sum []byte, releaseTime time.Time, rel
 func getUpdateTransport(timeout time.Duration) http.RoundTripper {
 	var updateTransport http.RoundTripper = &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(timeout),
+		DialContext:           xhttp.NewCustomDialContext(timeout, globalTCPOptions),
 		IdleConnTimeout:       timeout,
 		TLSHandshakeTimeout:   timeout,
 		ExpectContinueTimeout: timeout,

cmd/utils.go

@@ -574,6 +574,7 @@ func GetDefaultConnSettings() xhttp.ConnSettings {
 		DNSCache:    globalDNSCache,
 		DialTimeout: rest.DefaultTimeout,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 	}
 }
 
@@ -587,6 +588,7 @@ func NewInternodeHTTPTransport() func() http.RoundTripper {
 		CipherSuites:     fips.TLSCiphers(),
 		CurvePreferences: fips.TLSCurveIDs(),
 		EnableHTTP2:      false,
+		TCPOptions:       globalTCPOptions,
 	}.NewInternodeHTTPTransport()
 }
 
@@ -600,6 +602,7 @@ func NewCustomHTTPProxyTransport() func() *http.Transport {
 		CipherSuites:     fips.TLSCiphers(),
 		CurvePreferences: fips.TLSCurveIDs(),
 		EnableHTTP2:      false,
+		TCPOptions:       globalTCPOptions,
 	}.NewCustomHTTPProxyTransport()
 }
 
@@ -610,6 +613,7 @@ func NewHTTPTransportWithClientCerts(clientCert, clientKey string) *http.Transpo
 		DNSCache:    globalDNSCache,
 		DialTimeout: defaultDialTimeout,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 		EnableHTTP2: false,
 	}
 
@@ -643,6 +647,7 @@ func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport {
 		DNSCache:    globalDNSCache,
 		DialTimeout: defaultDialTimeout,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 		EnableHTTP2: false,
 	}.NewHTTPTransportWithTimeout(timeout)
 }
@@ -677,6 +682,7 @@ func NewRemoteTargetHTTPTransport() func() *http.Transport {
 		DialContext: newCustomDialContext(),
 		DNSCache:    globalDNSCache,
 		RootCAs:     globalRootCAs,
+		TCPOptions:  globalTCPOptions,
 		EnableHTTP2: false,
 	}.NewRemoteTargetHTTPTransport()
 }
@@ -1031,6 +1037,13 @@ func decodeDirObject(object string) string {
 	return object
 }
 
+func isDirObject(object string) bool {
+	if obj := encodeDirObject(object); obj != object {
+		object = obj
+	}
+	return HasSuffix(object, globalDirSuffix)
+}
+
 // Helper method to return total number of nodes in cluster
 func totalNodeCount() uint64 {
 	peers, _ := globalEndpoints.peers()

cmd/xl-storage-disk-id-check.go

@@ -781,7 +781,7 @@ func (p *xlStorageDiskIDCheck) checkHealth(ctx context.Context) (err error) {
 	t = time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess)))
 	if t > maxTimeSinceLastSuccess {
 		if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
-			logger.LogAlwaysIf(ctx, fmt.Errorf("taking drive %s offline, time since last response %v", p.storage.String(), t.Round(time.Millisecond)))
+			logger.LogAlwaysIf(ctx, fmt.Errorf("node(%s): taking drive %s offline, time since last response %v", globalLocalNodeName, p.storage.String(), t.Round(time.Millisecond)))
 			go p.monitorDiskStatus()
 		}
 		return errFaultyDisk
@@ -813,7 +813,7 @@ func (p *xlStorageDiskIDCheck) monitorDiskStatus() {
 			Force:     false,
 		})
 		if err == nil {
-			logger.Info("Able to read+write+delete, bringing drive %s online. Drive was offline for %s.", p.storage.String(),
+			logger.Info("node(%s): Read/Write/Delete successful, bringing drive %s online. Drive was offline for %s.", globalLocalNodeName, p.storage.String(),
 				time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess))))
 			atomic.StoreInt32(&p.health.status, diskHealthOK)
 			return

cmd/xl-storage.go

@@ -267,7 +267,8 @@ func newXLStorage(ep Endpoint, cleanUp bool) (s *xlStorage, err error) {
 	if err := s.checkODirectDiskSupport(); err == nil {
 		s.oDirect = true
 	} else {
-		if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+		// Allow if unsupported platform or single disk.
+		if errors.Is(err, errUnsupportedDisk) && globalIsErasureSD || !disk.ODirectPlatform {
 			s.oDirect = false
 		} else {
 			return s, err
@@ -375,6 +376,10 @@ func (s *xlStorage) Healing() *healingTracker {
 // with O_DIRECT support, return an error if any and return
 // errUnsupportedDisk if there is no O_DIRECT support
 func (s *xlStorage) checkODirectDiskSupport() error {
+	if !disk.ODirectPlatform {
+		return errUnsupportedDisk
+	}
+
 	// Check if backend is writable and supports O_DIRECT
 	uuid := mustGetUUID()
 	filePath := pathJoin(s.diskPath, ".writable-check-"+uuid+".tmp")
@@ -607,7 +612,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 
 // DiskInfo provides current information about disk space usage,
 // total free inodes and underlying filesystem.
-func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
+func (s *xlStorage) DiskInfo(_ context.Context) (info DiskInfo, err error) {
 	s.diskInfoCache.Once.Do(func() {
 		s.diskInfoCache.TTL = time.Second
 		s.diskInfoCache.Update = func() (interface{}, error) {
@@ -673,12 +678,12 @@ func (s *xlStorage) checkFormatJSON() (os.FileInfo, error) {
 			} else if osIsPermission(err) {
 				return nil, errDiskAccessDenied
 			}
-			logger.LogIf(GlobalContext, err) // log unexpected errors
+			logger.LogOnceIf(GlobalContext, err, err.Error()) // log unexpected errors
 			return nil, errCorruptedFormat
 		} else if osIsPermission(err) {
 			return nil, errDiskAccessDenied
 		}
-		logger.LogIf(GlobalContext, err) // log unexpected errors
+		logger.LogOnceIf(GlobalContext, err, err.Error()) // log unexpected errors
 		return nil, errCorruptedFormat
 	}
 	return fi, nil
@@ -1498,16 +1503,23 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		}
 		return nil, dmTime, err
 	}
-	r := &xioutil.ODirectReader{
-		File:      f,
-		SmallFile: true,
+	r := io.Reader(f)
+	var dr *xioutil.ODirectReader
+	if odirectEnabled {
+		dr = &xioutil.ODirectReader{
+			File:      f,
+			SmallFile: true,
+		}
+		defer dr.Close()
+		r = dr
+	} else {
+		defer f.Close()
 	}
-	defer r.Close()
 
 	// Get size for precise allocation.
 	stat, err := f.Stat()
 	if err != nil {
-		buf, err = io.ReadAll(r)
+		buf, err = io.ReadAll(diskHealthReader(ctx, r))
 		return buf, dmTime, osErrToFileErr(err)
 	}
 	if stat.IsDir() {
@@ -1522,6 +1534,9 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 	} else {
 		buf = make([]byte, sz)
 	}
+	if dr != nil {
+		dr.SmallFile = sz <= xioutil.BlockSizeSmall*2
+	}
 	// Read file...
 	_, err = io.ReadFull(diskHealthReader(ctx, r), buf)
 

docs/bucket/replication/setup_2site_existing_replication.sh

@@ -132,4 +132,76 @@ if [ $ret -ne 0 ]; then
     exit 1
 fi
 
+./mc mb sitea/bucket-version/
+./mc mb siteb/bucket-version
+
+./mc version enable sitea/bucket-version/
+./mc version enable siteb/bucket-version/
+
+echo "adding replication rule for site a -> site b"
+./mc replicate add sitea/bucket-version/ \
+     --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket-version
+
+./mc mb sitea/bucket-version/directory/
+
+sleep 2s
+
+./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt
+./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt
+
+out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
+fi
+
+./mc rm -r --versions --force sitea/bucket-version/
+
+sleep 2s
+
+./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt
+./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt
+
+out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
+fi
+
+## check if we don't create delete markers on the directory objects, its always permanent delete.
+./mc mb sitea/bucket-version/directory/
+
+sleep 2s
+
+./mc rm -r --force sitea/bucket-version/
+
+sleep 2s
+
+./mc ls -r --versions sitea/bucket-version/ > /tmp/sitea_dirs.txt
+./mc ls -r --versions siteb/bucket-version/ > /tmp/siteb_dirs.txt
+
+out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
+fi
+
+sitea_count=$(cat /tmp/sitea_dirs.txt | wc -l) # need to do it this way to avoid filename in the output
+siteb_count=$(cat /tmp/siteb_dirs.txt | wc -l) # need to do it this way to avoid filename in the output
+sitea_out=$(cat /tmp/sitea_dirs.txt)
+siteb_out=$(cat /tmp/siteb_dirs.txt)
+
+if [ $sitea_count -ne 0 ]; then
+    echo "BUG: expected no 'directory objects' left after deletion: ${sitea_out}"
+    exit 1
+fi
+
+if [ $siteb_count -ne 0 ]; then
+    echo "BUG: expected no 'directory objects' left after deletion: ${siteb_out}"
+    exit 1
+fi
+
 catch

docs/bucket/versioning/README.md

@@ -65,6 +65,18 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen
 
 ## MinIO extension to Bucket Versioning
 
+### Idempotent versions on directory objects
+
+All directory objects such as objects that end with `/`, will only have one versionId (i.e `null`). A delete marker will never be created on these directory objects, instead a DELETE will delete the directory objects. This is done to ensure that directory objects even with multiple overwrites - do not ever need multiple versions in the first place. All overwrite calls on these directory objects are idempotent.
+
+> NOTE: Server side replication is supported for idempotent versions on directory objects.
+
+### Idempotent versions on delete markers
+
+Duplicate delete markers are not created on MinIO buckets with versioning, if an application performs a soft delete on an object repeatedly - that object will only ever have a single DELETE marker for all such successive attempts. This is done to ensure that repeated soft deletes do not ever need multiple versions in the first place.
+
+> NOTE: Server side replication is supported for idempotent versions on delete marked objects.
+
 ### Motivation
 
 **PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on <https://subnet.min.io>.**
@@ -78,8 +90,6 @@ To exclude objects under a list of prefix (glob) patterns from being versioned,
 ```
 <VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
         <Status>Enabled</Status>
-        <ExcludeFolders>true</ExcludeFolders>
-
         <ExcludedPrefixes>
           <Prefix>*/_temporary</Prefix>
         </ExcludedPrefixes>

docs/metrics/prometheus/grafana/minio-dashboard.json

@@ -1742,98 +1742,6 @@
         "align": false
       }
     },
-    {
-      "aliasColors": {
-        "S3 Errors": "light-red",
-        "S3 Requests": "light-green"
-      },
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${DS_PROMETHEUS}"
-      },
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 10,
-        "w": 12,
-        "x": 12,
-        "y": 12
-      },
-      "hiddenSeries": false,
-      "id": 71,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
-      "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "9.0.0",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "exemplar": true,
-          "expr": "sum by (server,api) (increase(minio_s3_requests_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
-          "interval": "1m",
-          "intervalFactor": 2,
-          "legendFormat": "{{server,api}}",
-          "refId": "A",
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS}"
-          }
-        }
-      ],
-      "thresholds": [],
-      "timeRegions": [],
-      "title": "S3 API Request Error Rate",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "mode": "time",
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:331",
-          "format": "none",
-          "logBase": 1,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:332",
-          "format": "short",
-          "logBase": 1,
-          "show": false
-        }
-      ],
-      "yaxis": {
-        "align": false
-      }
-    },
     {
       "datasource": {
         "type": "prometheus",
@@ -2088,7 +1996,7 @@
       ],
       "thresholds": [],
       "timeRegions": [],
-      "title": "S3 API Request Error Rate (5xx)",
+      "title": "S3 API Request 5xx Error Rate",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -2180,7 +2088,7 @@
       ],
       "thresholds": [],
       "timeRegions": [],
-      "title": "S3 API Request Error Rate (4xx)",
+      "title": "S3 API Request 4xx Error Rate",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -2326,6 +2234,107 @@
         "align": false
       }
     },
+    {
+      "aliasColors": {
+      },
+      "bars": true,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "oQpdp674k"
+      },
+      "description": "Total TCP dial timeout/errors",
+      "fieldConfig": {
+        "defaults": {
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 10,
+      "fillGradient": 1,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 42
+      },
+      "hiddenSeries": false,
+      "id": 93,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "9.4.7",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "oQpdp674k"
+          },
+          "editorMode": "code",
+          "expr": "rate(minio_inter_node_traffic_dial_errors{job=\"$scrape_jobs\"}[$__rate_interval])",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Internode TCP errors",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:211",
+          "decimals": 0,
+          "format": "none",
+          "label": "TCP Errors",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:212",
+          "format": "s",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
     {
       "aliasColors": {},
       "bars": false,
@@ -2750,7 +2759,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "minio_cluster_disk_free_inodes{job=\"$scrape_jobs\"}",
+          "expr": "minio_node_disk_free_inodes{job=\"$scrape_jobs\"}",
           "format": "time_series",
           "instant": false,
           "interval": "",

docs/metrics/prometheus/grafana/minio-replication.json

@@ -2392,7 +2392,7 @@
         "targets": [
           {
             "exemplar": true,
-            "expr": "minio_cluster_disk_free_inodes{job=\"$scrape_jobs\"}",
+            "expr": "minio_node_disk_free_inodes{job=\"$scrape_jobs\"}",
             "format": "time_series",
             "instant": false,
             "interval": "",
@@ -2834,4 +2834,3 @@
     "uid": "TgmJnqnnk",
     "version": 18
   }
-  

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2023-04-13T03-08-07Z
+  image: quay.io/minio/minio:RELEASE.2023-04-28T18-11-17Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/cheggaaa/pb v1.0.29
 	github.com/coredns/coredns v1.10.1
 	github.com/coreos/go-oidc v2.2.1+incompatible
+	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dchest/siphash v1.2.3
 	github.com/djherbis/atime v1.1.0
@@ -32,7 +33,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/inconshreveable/mousetrap v1.1.0
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.16.4
+	github.com/klauspost/compress v1.16.5
 	github.com/klauspost/cpuid/v2 v2.2.4
 	github.com/klauspost/filepathx v1.1.1
 	github.com/klauspost/pgzip v1.2.5
@@ -42,15 +43,15 @@ require (
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/miekg/dns v1.1.53
 	github.com/minio/cli v1.24.2
-	github.com/minio/console v0.26.3
+	github.com/minio/console v0.26.4
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes-go v0.1.0
-	github.com/minio/madmin-go/v2 v2.0.20
+	github.com/minio/madmin-go/v2 v2.1.0
 	github.com/minio/minio-go/v7 v7.0.52
 	github.com/minio/mux v1.9.0
-	github.com/minio/pkg v1.6.6-0.20230330040824-5db111e5f63c
+	github.com/minio/pkg v1.7.1
 	github.com/minio/selfupdate v0.6.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.4.5
@@ -66,7 +67,7 @@ require (
 	github.com/philhofer/fwd v1.1.2
 	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/pkg/sftp v1.10.1
+	github.com/pkg/sftp v1.13.1
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
 	github.com/prometheus/common v0.42.0
@@ -112,7 +113,6 @@ require (
 	github.com/charmbracelet/lipgloss v0.7.1 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -129,8 +129,8 @@ require (
 	github.com/go-openapi/errors v0.20.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/runtime v0.25.0 // indirect
-	github.com/go-openapi/spec v0.20.8 // indirect
+	github.com/go-openapi/runtime v0.26.0 // indirect
+	github.com/go-openapi/spec v0.20.9 // indirect
 	github.com/go-openapi/strfmt v0.21.7 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-openapi/validate v0.22.1 // indirect
@@ -177,7 +177,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/colorjson v1.0.4 // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20230411170328-83336dfab325 // indirect
+	github.com/minio/mc v0.0.0-20230421183052-0da22db3af01 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/websocket v1.6.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -205,7 +205,7 @@ require (
 	github.com/rivo/tview v0.0.0-20230406072732-e22ce9588bb4 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rjeczalik/notify v0.9.3 // indirect
-	github.com/rs/xid v1.4.0 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.5 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect

go.sum

@@ -403,12 +403,12 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En
 github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g=
 github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro=
 github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw=
-github.com/go-openapi/runtime v0.25.0 h1:7yQTCdRbWhX8vnIjdzU8S00tBYf7Sg71EBeorlPHvhc=
-github.com/go-openapi/runtime v0.25.0/go.mod h1:Ux6fikcHXyyob6LNWxtE96hWwjBPYF0DXgVFuMTneOs=
+github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc=
+github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ=
 github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
 github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
-github.com/go-openapi/spec v0.20.8 h1:ubHmXNY3FCIOinT8RNrrPfGc9t7I1qhPtdOGoG2AxRU=
-github.com/go-openapi/spec v0.20.8/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
+github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8=
+github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
 github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
 github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
 github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg=
@@ -663,8 +663,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU=
-github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
+github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
@@ -765,8 +765,8 @@ github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
 github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.4 h1:sNJYTb2uNswdqmGARg9wrogCX8+GRZzEacYbJT86e00=
 github.com/minio/colorjson v1.0.4/go.mod h1:ZgE8vYon4xC4yfBPclP/2gqMRYw+p+xRsBbLMDKdb9M=
-github.com/minio/console v0.26.3 h1:RkDfgt/DlOcol+3n0xUD6W+WuRGN23+PkrkeOfL75rw=
-github.com/minio/console v0.26.3/go.mod h1:aAdvnrn+2SLArb2HtauEP7u+eMLmvFhA7nUIsYSD8iI=
+github.com/minio/console v0.26.4 h1:IXvIVC/LSNt3YLmsblQECq++bBtfOfWFeBM8VvagaNU=
+github.com/minio/console v0.26.4/go.mod h1:qgFWVTSDx9CUIIkfsPUET0MA+ll1t8CiBkXtqZS4J6g=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dperf v0.4.2 h1:u2fcdXViX3+alZcEYSOh1crNbGtVU5drQ24lP3Xx6j8=
@@ -778,12 +778,10 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
 github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc=
 github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo=
 github.com/minio/madmin-go v1.6.6/go.mod h1:ATvkBOLiP3av4D++2v1UEHC/QzsGtgXD5kYvvRYzdKs=
-github.com/minio/madmin-go/v2 v2.0.19 h1:XznxdMVCTyr0A88JrZFhdxWY8KLfJcrs0TTmFiE9cc8=
-github.com/minio/madmin-go/v2 v2.0.19/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE=
-github.com/minio/madmin-go/v2 v2.0.20 h1:EO2IIQsnaVM3ki/ONcW0Ry4UpDn5aa+/S1kLkHuDBs8=
-github.com/minio/madmin-go/v2 v2.0.20/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE=
-github.com/minio/mc v0.0.0-20230411170328-83336dfab325 h1:da5I7G0Va6UnqoxzPHus7zmqo2pEQnltI09/XKNvHP4=
-github.com/minio/mc v0.0.0-20230411170328-83336dfab325/go.mod h1:v9AeUV4eaMpKCuNz0tk/MSGxvNs3duYvWUIxzpacxtc=
+github.com/minio/madmin-go/v2 v2.1.0 h1:d2n2RsOLiP4de0FxnsHCr4gACZVm5bMjzroz6T0QrO0=
+github.com/minio/madmin-go/v2 v2.1.0/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE=
+github.com/minio/mc v0.0.0-20230421183052-0da22db3af01 h1:8o0xnhhmVlFDi+BMYib5EL5cgBD1z5M2Wk8RrZF6RFg=
+github.com/minio/mc v0.0.0-20230421183052-0da22db3af01/go.mod h1:v9AeUV4eaMpKCuNz0tk/MSGxvNs3duYvWUIxzpacxtc=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
@@ -793,8 +791,8 @@ github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9
 github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA=
 github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ=
 github.com/minio/pkg v1.5.4/go.mod h1:2MOaRFdmFKULD+uOLc3qHLGTQTuxCNPKNPfLBTxC8CA=
-github.com/minio/pkg v1.6.6-0.20230330040824-5db111e5f63c h1:Ukw0+d0T/+9lserJfodr4HGIIb3hLkt8tlgMh2vUfZg=
-github.com/minio/pkg v1.6.6-0.20230330040824-5db111e5f63c/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=
+github.com/minio/pkg v1.7.1 h1:Nu5EJ64PHmJtbq1zOxQv2lNM6drjbhO8QOCdntHSjVE=
+github.com/minio/pkg v1.7.1/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=
 github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
 github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -899,8 +897,9 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
-github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/pkg/sftp v1.13.1 h1:I2qBYMChEhIjOgazfJmV3/mZM256btk6wkCDRmW7JYs=
+github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
 github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
 github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -970,8 +969,9 @@ github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo=
 github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -1144,6 +1144,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
@@ -1365,6 +1366,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 5.0.8
-appVersion: RELEASE.2023-04-13T03-08-07Z
+version: 5.0.9
+appVersion: RELEASE.2023-04-28T18-11-17Z
 keywords:
   - minio
   - storage

helm/minio/templates/console-ingress.yaml

@@ -36,20 +36,20 @@ spec:
     - http:
         paths:
           - path: {{ $ingressPath }}
-          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+            {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
             pathType: Prefix
             backend:
               service:
                 name: {{ $fullName }}
                 port: 
                   number: {{ $servicePort }}
-          {{- else }}
+            {{- else }}
             backend:
               serviceName: {{ $fullName }}
               servicePort: {{ $servicePort }}
-          {{- end }}
+            {{- end }}
       {{- if . }}
-      host: {{ . | quote }}
+      host: {{ tpl . $ | quote }}
       {{- end }}
     {{- end }}
 {{- end }}

helm/minio/templates/deployment.yaml

@@ -18,11 +18,11 @@ metadata:
   {{- end }}
 spec:
   strategy:
-    type: {{ .Values.DeploymentUpdate.type }}
-    {{- if eq .Values.DeploymentUpdate.type "RollingUpdate" }}
+    type: {{ .Values.deploymentUpdate.type }}
+    {{- if eq .Values.deploymentUpdate.type "RollingUpdate" }}
     rollingUpdate:
-      maxSurge: {{ .Values.DeploymentUpdate.maxSurge }}
-      maxUnavailable: {{ .Values.DeploymentUpdate.maxUnavailable }}
+      maxSurge: {{ .Values.deploymentUpdate.maxSurge }}
+      maxUnavailable: {{ .Values.deploymentUpdate.maxUnavailable }}
     {{- end }}
   replicas: 1
   selector:
@@ -154,7 +154,7 @@ spec:
             {{- end }}
             {{- range $key, $val := .Values.environment }}
             - name: {{ $key }}
-              value: {{ $val | quote }}
+              value: {{ tpl $val $ | quote }}
             {{- end }}
           resources: {{- toYaml .Values.resources | nindent 12 }}
         {{- with .Values.extraContainers }}

helm/minio/templates/ingress.yaml

@@ -49,7 +49,7 @@ spec:
               servicePort: {{ $servicePort }}
             {{- end }}
       {{- if . }}
-      host: {{ . | quote }}
+      host: {{ tpl . $ | quote }}
       {{- end }}
     {{- end }}
 {{- end }}

helm/minio/templates/pvc.yaml

@@ -25,8 +25,8 @@ spec:
   storageClassName: "{{ .Values.persistence.storageClass }}"
   {{- end }}
   {{- end }}
-  {{- if .Values.persistence.VolumeName }}
-  volumeName: "{{ .Values.persistence.VolumeName }}"
+  {{- if .Values.persistence.volumeName }}
+  volumeName: "{{ .Values.persistence.volumeName }}"
   {{- end }}
 {{- end }}
 {{- end }}

helm/minio/templates/statefulset.yaml

@@ -49,7 +49,7 @@ metadata:
   {{- end }}
 spec:
   updateStrategy:
-    type: {{ .Values.StatefulSetUpdate.updateStrategy }}
+    type: {{ .Values.statefulSetUpdate.updateStrategy }}
   podManagementPolicy: "Parallel"
   serviceName: {{ template "minio.fullname" . }}-svc
   replicas: {{ $replicas }}
@@ -172,7 +172,7 @@ spec:
             {{- end }}
             {{- range $key, $val := .Values.environment }}
             - name: {{ $key }}
-              value: {{ $val | quote }}
+              value: {{ tpl $val $ | quote }}
             {{- end }}
           resources: {{- toYaml .Values.resources | nindent 12 }}
         {{- with .Values.extraContainers }}

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2023-04-13T03-08-07Z
+  tag: RELEASE.2023-04-28T18-11-17Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -64,13 +64,13 @@ minioAPIPort: "9000"
 minioConsolePort: "9001"
 
 ## Update strategy for Deployments
-DeploymentUpdate:
+deploymentUpdate:
   type: RollingUpdate
   maxUnavailable: 0
   maxSurge: 100%
 
 ## Update strategy for StatefulSets
-StatefulSetUpdate:
+statefulSetUpdate:
   updateStrategy: RollingUpdate
 
 ## Pod priority settings
@@ -157,7 +157,7 @@ persistence:
   ## Storage class of PV to bind. By default it looks for standard storage class.
   ## If the PV uses a different storage class, specify that here.
   storageClass: ""
-  VolumeName: ""
+  volumeName: ""
   accessMode: ReadWriteOnce
   size: 500Gi
 

index.yaml

@@ -1,9 +1,31 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2023-04-28T18-11-17Z
+    created: "2023-05-02T23:22:52.717714795-07:00"
+    description: Multi-Cloud Object Storage
+    digest: cf98985e32675e4ce327304ea9ac61046a788b3d5190d6b501330f7803d41a11
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-5.0.9.tgz
+    version: 5.0.9
   - apiVersion: v1
     appVersion: RELEASE.2023-04-13T03-08-07Z
-    created: "2023-04-13T14:39:33.904722194-07:00"
+    created: "2023-05-02T23:22:52.714055612-07:00"
     description: Multi-Cloud Object Storage
     digest: 034d68f85799f6693836975797f85a91842cf2d003a6c4ff401bd4ea4c946af6
     home: https://min.io
@@ -25,7 +47,7 @@ entries:
     version: 5.0.8
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-04-13T14:39:33.902972411-07:00"
+    created: "2023-05-02T23:22:52.71067834-07:00"
     description: Multi-Cloud Object Storage
     digest: 3f935a310e1b5b873052629b66005c160356ca7b2bd394cb07b34dbaf9905e3f
     home: https://min.io
@@ -47,7 +69,7 @@ entries:
     version: 5.0.7
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-04-13T14:39:33.899768149-07:00"
+    created: "2023-05-02T23:22:52.707176759-07:00"
     description: Multi-Cloud Object Storage
     digest: 82ef858ce483c2d736444792986cb36bd0fb4fc90a80b97fe30d7b2f2034d24a
     home: https://min.io
@@ -69,7 +91,7 @@ entries:
     version: 5.0.6
   - apiVersion: v1
     appVersion: RELEASE.2023-01-31T02-24-19Z
-    created: "2023-04-13T14:39:33.898010335-07:00"
+    created: "2023-05-02T23:22:52.70251238-07:00"
     description: Multi-Cloud Object Storage
     digest: fefeea10e4e525e45f82fb80a03900d34605ec432dd92f56d94eaf4fb1b98c41
     home: https://min.io
@@ -91,7 +113,7 @@ entries:
     version: 5.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-04-13T14:39:33.896365088-07:00"
+    created: "2023-05-02T23:22:52.699027922-07:00"
     description: Multi-Cloud Object Storage
     digest: 6b305783c98b0b97ffab079ff4430094fd0ca6e98e82bb8153cb93033a1bf40f
     home: https://min.io
@@ -113,7 +135,7 @@ entries:
     version: 5.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-04-13T14:39:33.894702003-07:00"
+    created: "2023-05-02T23:22:52.694723934-07:00"
     description: Multi-Cloud Object Storage
     digest: bac89157c53b324aece263c294aa49f5c9b64f426b4b06c9bca3d72e77e244f2
     home: https://min.io
@@ -135,7 +157,7 @@ entries:
     version: 5.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-04-13T14:39:33.89270488-07:00"
+    created: "2023-05-02T23:22:52.69140315-07:00"
     description: Multi-Cloud Object Storage
     digest: 935ce4f09366231b11d414d626f887fa6fa6024dd30a42e81e810ca1438d5904
     home: https://min.io
@@ -157,7 +179,7 @@ entries:
     version: 5.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-11-11T03-44-20Z
-    created: "2023-04-13T14:39:33.89095156-07:00"
+    created: "2023-05-02T23:22:52.687984634-07:00"
     description: Multi-Cloud Object Storage
     digest: 3e952c5d737980b8ccdfb819021eafb4b4e8da226f764a1dc3de1ba63ceb1ffa
     home: https://min.io
@@ -179,7 +201,7 @@ entries:
     version: 5.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-04-13T14:39:33.88770083-07:00"
+    created: "2023-05-02T23:22:52.683331965-07:00"
     description: Multi-Cloud Object Storage
     digest: 6215c800d84fd4c40e4fb4142645fc1c6a039c251776a3cc8c11a24b9e3b59c7
     home: https://min.io
@@ -201,7 +223,7 @@ entries:
     version: 5.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-04-13T14:39:33.885874401-07:00"
+    created: "2023-05-02T23:22:52.679468114-07:00"
     description: Multi-Cloud Object Storage
     digest: 2d3d884490ea1127742f938bc9382844bae713caae08b3308f766f3c9000659a
     home: https://min.io
@@ -223,7 +245,7 @@ entries:
     version: 4.1.0
   - apiVersion: v1
     appVersion: RELEASE.2022-09-17T00-09-45Z
-    created: "2023-04-13T14:39:33.870938962-07:00"
+    created: "2023-05-02T23:22:52.644057838-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f16f2dbfed91ab81a7fae60b6ea32f554365bd27bf5fda55b64a0fa264f4252
     home: https://min.io
@@ -245,7 +267,7 @@ entries:
     version: 4.0.15
   - apiVersion: v1
     appVersion: RELEASE.2022-09-01T23-53-36Z
-    created: "2023-04-13T14:39:33.869292215-07:00"
+    created: "2023-05-02T23:22:52.639559167-07:00"
     description: Multi-Cloud Object Storage
     digest: 35d89d8f49d53ea929466fb88ee26123431326033f1387e6b2d536a629c0a398
     home: https://min.io
@@ -267,7 +289,7 @@ entries:
     version: 4.0.14
   - apiVersion: v1
     appVersion: RELEASE.2022-08-22T23-53-06Z
-    created: "2023-04-13T14:39:33.867593706-07:00"
+    created: "2023-05-02T23:22:52.63647284-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b86937ca88d9f6046141fdc2b1cc54760435ed92d289cd0a115fa7148781d4e
     home: https://min.io
@@ -289,7 +311,7 @@ entries:
     version: 4.0.13
   - apiVersion: v1
     appVersion: RELEASE.2022-08-13T21-54-44Z
-    created: "2023-04-13T14:39:33.86487566-07:00"
+    created: "2023-05-02T23:22:52.633371904-07:00"
     description: Multi-Cloud Object Storage
     digest: 2d9c227c0f46ea8bdef4d760c212156fd4c6623ddc5406779c569fe925527787
     home: https://min.io
@@ -311,7 +333,7 @@ entries:
     version: 4.0.12
   - apiVersion: v1
     appVersion: RELEASE.2022-08-05T23-27-09Z
-    created: "2023-04-13T14:39:33.863292598-07:00"
+    created: "2023-05-02T23:22:52.630331186-07:00"
     description: Multi-Cloud Object Storage
     digest: 6caaffcb636e040cd7e8bc4883a1674a673757f4781c32d53b5ec0f41fea3944
     home: https://min.io
@@ -333,7 +355,7 @@ entries:
     version: 4.0.11
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-04-13T14:39:33.861701816-07:00"
+    created: "2023-05-02T23:22:52.627209812-07:00"
     description: Multi-Cloud Object Storage
     digest: 841d87788fb094d6a7d8a91e91821fe1e847bc952e054c781fc93742d112e18a
     home: https://min.io
@@ -355,7 +377,7 @@ entries:
     version: 4.0.10
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-04-13T14:39:33.884239166-07:00"
+    created: "2023-05-02T23:22:52.67512565-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f1a78382df3215deac07495a5e7de7009a1153b4cf6cb565630652a69aec4cf
     home: https://min.io
@@ -377,7 +399,7 @@ entries:
     version: 4.0.9
   - apiVersion: v1
     appVersion: RELEASE.2022-07-29T19-40-48Z
-    created: "2023-04-13T14:39:33.882599235-07:00"
+    created: "2023-05-02T23:22:52.669678613-07:00"
     description: Multi-Cloud Object Storage
     digest: d11db37963636922cb778b6bc0ad2ca4724cb391ea7b785995ada52467d7dd83
     home: https://min.io
@@ -399,7 +421,7 @@ entries:
     version: 4.0.8
   - apiVersion: v1
     appVersion: RELEASE.2022-07-26T00-53-03Z
-    created: "2023-04-13T14:39:33.881089691-07:00"
+    created: "2023-05-02T23:22:52.664066124-07:00"
     description: Multi-Cloud Object Storage
     digest: ca775e08c84331bb5029d4d29867d30c16e2c62e897788eb432212a756e91e4e
     home: https://min.io
@@ -421,7 +443,7 @@ entries:
     version: 4.0.7
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-04-13T14:39:33.879531106-07:00"
+    created: "2023-05-02T23:22:52.659665559-07:00"
     description: Multi-Cloud Object Storage
     digest: 06542b8f3d149d5908b15de9a8d6f8cf304af0213830be56dc315785d14f9ccd
     home: https://min.io
@@ -443,7 +465,7 @@ entries:
     version: 4.0.6
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-04-13T14:39:33.877099765-07:00"
+    created: "2023-05-02T23:22:52.656625056-07:00"
     description: Multi-Cloud Object Storage
     digest: dd2676362f067454a496cdd293609d0c904b08f521625af49f95402a024ba1f5
     home: https://min.io
@@ -465,7 +487,7 @@ entries:
     version: 4.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-04-13T14:39:33.875614067-07:00"
+    created: "2023-05-02T23:22:52.653668918-07:00"
     description: Multi-Cloud Object Storage
     digest: bab9ef192d4eda4c572ad0ce0cf551736c847f582d1837d6833ee10543c23167
     home: https://min.io
@@ -487,7 +509,7 @@ entries:
     version: 4.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-04-13T14:39:33.874105602-07:00"
+    created: "2023-05-02T23:22:52.650656104-07:00"
     description: Multi-Cloud Object Storage
     digest: c770bb9841c76576e4e8573f78b0ec33e0d729504c9667e67ad62d48df5ed64c
     home: https://min.io
@@ -509,7 +531,7 @@ entries:
     version: 4.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-04-13T14:39:33.872532023-07:00"
+    created: "2023-05-02T23:22:52.647527303-07:00"
     description: Multi-Cloud Object Storage
     digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb
     home: https://min.io
@@ -531,7 +553,7 @@ entries:
     version: 4.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-04-30T22-23-53Z
-    created: "2023-04-13T14:39:33.860146689-07:00"
+    created: "2023-05-02T23:22:52.623126646-07:00"
     description: Multi-Cloud Object Storage
     digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f
     home: https://min.io
@@ -553,7 +575,7 @@ entries:
     version: 4.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-04-26T01-20-24Z
-    created: "2023-04-13T14:39:33.85863375-07:00"
+    created: "2023-05-02T23:22:52.619692687-07:00"
     description: Multi-Cloud Object Storage
     digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2
     home: https://min.io
@@ -575,7 +597,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-04-16T04-26-02Z
-    created: "2023-04-13T14:39:33.857146288-07:00"
+    created: "2023-05-02T23:22:52.615620936-07:00"
     description: Multi-Cloud Object Storage
     digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1
     home: https://min.io
@@ -597,7 +619,7 @@ entries:
     version: 3.6.6
   - apiVersion: v1
     appVersion: RELEASE.2022-04-12T06-55-35Z
-    created: "2023-04-13T14:39:33.855161049-07:00"
+    created: "2023-05-02T23:22:52.612395282-07:00"
     description: Multi-Cloud Object Storage
     digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650
     home: https://min.io
@@ -619,7 +641,7 @@ entries:
     version: 3.6.5
   - apiVersion: v1
     appVersion: RELEASE.2022-04-09T15-09-52Z
-    created: "2023-04-13T14:39:33.853131766-07:00"
+    created: "2023-05-02T23:22:52.609626098-07:00"
     description: Multi-Cloud Object Storage
     digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06
     home: https://min.io
@@ -641,7 +663,7 @@ entries:
     version: 3.6.4
   - apiVersion: v1
     appVersion: RELEASE.2022-03-24T00-43-44Z
-    created: "2023-04-13T14:39:33.851651471-07:00"
+    created: "2023-05-02T23:22:52.606608511-07:00"
     description: Multi-Cloud Object Storage
     digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75
     home: https://min.io
@@ -663,7 +685,7 @@ entries:
     version: 3.6.3
   - apiVersion: v1
     appVersion: RELEASE.2022-03-17T06-34-49Z
-    created: "2023-04-13T14:39:33.850155733-07:00"
+    created: "2023-05-02T23:22:52.602714345-07:00"
     description: Multi-Cloud Object Storage
     digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495
     home: https://min.io
@@ -685,7 +707,7 @@ entries:
     version: 3.6.2
   - apiVersion: v1
     appVersion: RELEASE.2022-03-14T18-25-24Z
-    created: "2023-04-13T14:39:33.848658622-07:00"
+    created: "2023-05-02T23:22:52.599800755-07:00"
     description: Multi-Cloud Object Storage
     digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
     home: https://min.io
@@ -707,7 +729,7 @@ entries:
     version: 3.6.1
   - apiVersion: v1
     appVersion: RELEASE.2022-03-11T23-57-45Z
-    created: "2023-04-13T14:39:33.8471424-07:00"
+    created: "2023-05-02T23:22:52.596063931-07:00"
     description: Multi-Cloud Object Storage
     digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
     home: https://min.io
@@ -729,7 +751,7 @@ entries:
     version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2023-04-13T14:39:33.845572238-07:00"
+    created: "2023-05-02T23:22:52.593272488-07:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -751,7 +773,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2023-04-13T14:39:33.843283699-07:00"
+    created: "2023-05-02T23:22:52.590564356-07:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -773,7 +795,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-04-13T14:39:33.841793213-07:00"
+    created: "2023-05-02T23:22:52.587730784-07:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -795,7 +817,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-04-13T14:39:33.840320255-07:00"
+    created: "2023-05-02T23:22:52.584116925-07:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -817,7 +839,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2023-04-13T14:39:33.838848475-07:00"
+    created: "2023-05-02T23:22:52.581284172-07:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -839,7 +861,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-04-13T14:39:33.837463455-07:00"
+    created: "2023-05-02T23:22:52.578447052-07:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -861,7 +883,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-04-13T14:39:33.836037131-07:00"
+    created: "2023-05-02T23:22:52.575538055-07:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -883,7 +905,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2023-04-13T14:39:33.834131809-07:00"
+    created: "2023-05-02T23:22:52.572954303-07:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -905,7 +927,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-04-13T14:39:33.832190048-07:00"
+    created: "2023-05-02T23:22:52.570302766-07:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -927,7 +949,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-04-13T14:39:33.829251194-07:00"
+    created: "2023-05-02T23:22:52.566413634-07:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -949,7 +971,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2023-04-13T14:39:33.827349955-07:00"
+    created: "2023-05-02T23:22:52.563847647-07:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -971,7 +993,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2023-04-13T14:39:33.825179668-07:00"
+    created: "2023-05-02T23:22:52.56118185-07:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -993,7 +1015,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2023-04-13T14:39:33.823029641-07:00"
+    created: "2023-05-02T23:22:52.558763511-07:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -1015,7 +1037,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2023-04-13T14:39:33.821363456-07:00"
+    created: "2023-05-02T23:22:52.556331196-07:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -1037,7 +1059,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2023-04-13T14:39:33.81968069-07:00"
+    created: "2023-05-02T23:22:52.553894726-07:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -1059,7 +1081,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-04-13T14:39:33.817648739-07:00"
+    created: "2023-05-02T23:22:52.551455123-07:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -1081,7 +1103,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-04-13T14:39:33.816178628-07:00"
+    created: "2023-05-02T23:22:52.547289276-07:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -1103,7 +1125,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-04-13T14:39:33.813912605-07:00"
+    created: "2023-05-02T23:22:52.544822046-07:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -1125,7 +1147,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2023-04-13T14:39:33.812658948-07:00"
+    created: "2023-05-02T23:22:52.542542681-07:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -1147,7 +1169,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-04-13T14:39:33.811338087-07:00"
+    created: "2023-05-02T23:22:52.540251751-07:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -1169,7 +1191,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-04-13T14:39:33.809986868-07:00"
+    created: "2023-05-02T23:22:52.538100641-07:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -1191,7 +1213,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-04-13T14:39:33.80878071-07:00"
+    created: "2023-05-02T23:22:52.535860995-07:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -1213,7 +1235,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-04-13T14:39:33.807416726-07:00"
+    created: "2023-05-02T23:22:52.531650814-07:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -1235,7 +1257,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-04-13T14:39:33.806163505-07:00"
+    created: "2023-05-02T23:22:52.529034314-07:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -1257,7 +1279,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2023-04-13T14:39:33.804903629-07:00"
+    created: "2023-05-02T23:22:52.526547269-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -1279,7 +1301,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2023-04-13T14:39:33.802626491-07:00"
+    created: "2023-05-02T23:22:52.524494789-07:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -1301,7 +1323,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2023-04-13T14:39:33.801365971-07:00"
+    created: "2023-05-02T23:22:52.522263819-07:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -1323,7 +1345,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2023-04-13T14:39:33.800136574-07:00"
+    created: "2023-05-02T23:22:52.520154865-07:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -1345,7 +1367,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-04-13T14:39:33.798923934-07:00"
+    created: "2023-05-02T23:22:52.518010066-07:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -1367,7 +1389,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-04-13T14:39:33.797532313-07:00"
+    created: "2023-05-02T23:22:52.514394186-07:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -1389,7 +1411,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-04-13T14:39:33.796286874-07:00"
+    created: "2023-05-02T23:22:52.511727867-07:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -1411,7 +1433,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-04-13T14:39:33.794957696-07:00"
+    created: "2023-05-02T23:22:52.509627921-07:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -1433,7 +1455,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-04-13T14:39:33.793014019-07:00"
+    created: "2023-05-02T23:22:52.507565595-07:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -1455,7 +1477,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-04-13T14:39:33.790939383-07:00"
+    created: "2023-05-02T23:22:52.505508391-07:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -1477,7 +1499,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-04-13T14:39:33.788074463-07:00"
+    created: "2023-05-02T23:22:52.503420894-07:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -1499,7 +1521,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2023-04-13T14:39:33.786791063-07:00"
+    created: "2023-05-02T23:22:52.501153476-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -1521,7 +1543,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-04-13T14:39:33.785648827-07:00"
+    created: "2023-05-02T23:22:52.497257542-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -1543,7 +1565,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-04-13T14:39:33.78451068-07:00"
+    created: "2023-05-02T23:22:52.494795961-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -1565,7 +1587,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-04-13T14:39:33.783363751-07:00"
+    created: "2023-05-02T23:22:52.492715441-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -1585,7 +1607,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-04-13T14:39:33.782197288-07:00"
+    created: "2023-05-02T23:22:52.490165864-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -1605,7 +1627,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-04-13T14:39:33.780959636-07:00"
+    created: "2023-05-02T23:22:52.487691026-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -1625,7 +1647,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-04-13T14:39:33.778433355-07:00"
+    created: "2023-05-02T23:22:52.484375697-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -1645,7 +1667,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-04-13T14:39:33.776484146-07:00"
+    created: "2023-05-02T23:22:52.481287814-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -1665,7 +1687,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-04-13T14:39:33.774527099-07:00"
+    created: "2023-05-02T23:22:52.475930421-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -1685,7 +1707,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-04-13T14:39:33.772578779-07:00"
+    created: "2023-05-02T23:22:52.473881221-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -1705,7 +1727,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2023-04-13T14:39:33.770495939-07:00"
+    created: "2023-05-02T23:22:52.47182851-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -1723,4 +1745,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2023-04-13T14:39:33.768251143-07:00"
+generated: "2023-05-02T23:22:52.469209433-07:00"

internal/bucket/lifecycle/lifecycle.go

@@ -287,39 +287,12 @@ func (o ObjectOpts) ExpiredObjectDeleteMarker() bool {
 
 // Event contains a lifecycle action with associated info
 type Event struct {
-	Action       Action
-	RuleID       string
-	Due          time.Time
-	StorageClass string
-}
-
-type lifecycleEvents []Event
-
-func (es lifecycleEvents) Len() int {
-	return len(es)
-}
-
-func (es lifecycleEvents) Swap(i, j int) {
-	es[i], es[j] = es[j], es[i]
-}
-
-func (es lifecycleEvents) Less(i, j int) bool {
-	if es[i].Due.Equal(es[j].Due) {
-		// Prefer Expiration over Transition for both current and noncurrent
-		// versions
-		switch es[i].Action {
-		case DeleteAction, DeleteVersionAction:
-			return true
-		}
-		switch es[j].Action {
-		case DeleteAction, DeleteVersionAction:
-			return false
-		}
-		return true
-	}
-
-	// Prefer earlier occurring event
-	return es[i].Due.Before(es[j].Due)
+	Action                  Action
+	RuleID                  string
+	Due                     time.Time
+	NoncurrentDays          int
+	NewerNoncurrentVersions int
+	StorageClass            string
 }
 
 // Eval returns the lifecycle event applicable now.
@@ -451,7 +424,24 @@ func (lc Lifecycle) eval(obj ObjectOpts, now time.Time) Event {
 	}
 
 	if len(events) > 0 {
-		sort.Sort(lifecycleEvents(events))
+		sort.Slice(events, func(i, j int) bool {
+			if events[i].Due.Equal(events[j].Due) {
+				// Prefer Expiration over Transition for both current
+				// and noncurrent versions
+				switch events[i].Action {
+				case DeleteAction, DeleteVersionAction:
+					return true
+				}
+				switch events[j].Action {
+				case DeleteAction, DeleteVersionAction:
+					return false
+				}
+				return true
+			}
+
+			// Prefer earlier occurring event
+			return events[i].Due.Before(events[j].Due)
+		})
 		return events[0]
 	}
 
@@ -492,15 +482,17 @@ func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts)
 
 // NoncurrentVersionsExpirationLimit returns the number of noncurrent versions
 // to be retained from the first applicable rule per S3 behavior.
-func (lc Lifecycle) NoncurrentVersionsExpirationLimit(obj ObjectOpts) (string, int, int) {
-	var lim int
-	var days int
-	var ruleID string
+func (lc Lifecycle) NoncurrentVersionsExpirationLimit(obj ObjectOpts) Event {
 	for _, rule := range lc.FilterRules(obj) {
 		if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions == 0 {
 			continue
 		}
-		return rule.ID, int(rule.NoncurrentVersionExpiration.NoncurrentDays), rule.NoncurrentVersionExpiration.NewerNoncurrentVersions
+		return Event{
+			Action:                  DeleteVersionAction,
+			RuleID:                  rule.ID,
+			NoncurrentDays:          int(rule.NoncurrentVersionExpiration.NoncurrentDays),
+			NewerNoncurrentVersions: rule.NoncurrentVersionExpiration.NewerNoncurrentVersions,
+		}
 	}
-	return ruleID, days, lim
+	return Event{}
 }

internal/bucket/lifecycle/lifecycle_test.go

@@ -854,8 +854,8 @@ func TestNoncurrentVersionsLimit(t *testing.T) {
 	lc := Lifecycle{
 		Rules: rules,
 	}
-	if ruleID, days, lim := lc.NoncurrentVersionsExpirationLimit(ObjectOpts{Name: "obj"}); ruleID != "1" || days != 1 || lim != 1 {
-		t.Fatalf("Expected (ruleID, days, lim) to be (\"1\", 1, 1) but got (%s, %d, %d)", ruleID, days, lim)
+	if event := lc.NoncurrentVersionsExpirationLimit(ObjectOpts{Name: "obj"}); event.RuleID != "1" || event.NoncurrentDays != 1 || event.NewerNoncurrentVersions != 1 {
+		t.Fatalf("Expected (ruleID, days, lim) to be (\"1\", 1, 1) but got (%s, %d, %d)", event.RuleID, event.NoncurrentDays, event.NewerNoncurrentVersions)
 	}
 }