fix: typo in FIPS sha256 (#15024)

Indexed streams would be decoded by the legacy loader if there 
was an error loading it. Return an error when the stream is indexed 
and it cannot be loaded.

Fixes "unknown minor metadata version" on corrupted xl.meta files and 
returns an actual error.

Makefile

@@ -53,7 +53,7 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
 	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
+	@GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
 test-replication: install ## verify multi site replication
 	@echo "Running tests for replicating three sites"
@@ -73,18 +73,18 @@ test-site-replication-minio: install ## verify automatic site replication
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"
-	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
-	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
 verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
-	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
 
 build: checks ## builds minio to $(PWD)

README.md

@@ -196,12 +196,6 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
 service iptables restart
 ```
 
-## Pre-existing data
-
-When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
-
-The above statement is also valid for all gateway backends.
-
 ## Test MinIO Connectivity
 
 ### Test using MinIO Console

buildscripts/race.sh

@@ -2,6 +2,7 @@
 
 set -e
 
+export GORACE="history_size=7"
 ## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
 for d in $(go list ./... | grep -v dsync); do
     CGO_ENABLED=1 go test -v -race --timeout 100m "$d"

cmd/admin-bucket-handlers.go

@@ -145,7 +145,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	bucket := pathClean(vars["bucket"])
 	update := r.Form.Get("update") == "true"
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -274,7 +274,8 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 	arnType := vars["type"]
-	if !globalIsErasure {
+
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -314,7 +315,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	bucket := pathClean(vars["bucket"])
 	arn := vars["arn"]
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}

cmd/admin-handlers-config-kv.go

@@ -33,6 +33,7 @@ import (
 	"github.com/minio/minio/internal/config/etcd"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
@@ -444,6 +445,8 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 				off = !xldap.Enabled(kv)
 			case config.IdentityTLSSubSys:
 				off = !globalSTSTLSConfig.Enabled
+			case config.IdentityPluginSubSys:
+				off = !idplugin.Enabled(kv)
 			}
 			if off {
 				s.WriteString(config.KvComment)

cmd/admin-handlers-users-race_test.go

@@ -47,10 +47,10 @@ func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) {
 	}
 
 	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on ErasureSD backend with signature v4.
+		{serverType: "ErasureSD", signer: signerV4},
+		// Init and run test on ErasureSD backend, with tls enabled.
+		{serverType: "ErasureSD", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
 		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.

cmd/admin-handlers-users_test.go

@@ -102,10 +102,10 @@ func (s *TestSuiteIAM) iamSetup(c *check) {
 // common to tests.
 var iamTestSuites = func() []*TestSuiteIAM {
 	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on ErasureSD backend with signature v4.
+		{serverType: "ErasureSD", signer: signerV4},
+		// Init and run test on ErasureSD backend, with tls enabled.
+		{serverType: "ErasureSD", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
 		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.

cmd/admin-handlers.go

@@ -805,8 +805,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Check if this setup has an erasure coded backend.
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
 		return
 	}
@@ -998,7 +997,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Check if this setup has an erasure coded backend.
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
 		return
 	}
@@ -1078,7 +1077,7 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1228,7 +1227,7 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}

cmd/admin-router.go

@@ -170,50 +170,48 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// Set Group Status
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetGroupStatus))).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
-		if globalIsDistErasure || globalIsErasure {
-			// GetBucketQuotaConfig
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
-			// PutBucketQuotaConfig
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+		// GetBucketQuotaConfig
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+		// PutBucketQuotaConfig
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
 
-			// Bucket replication operations
-			// GetBucketTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
-			// RemoveRemoteTargetHandler
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+		// Bucket replication operations
+		// GetBucketTargetHandler
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+		// SetRemoteTargetHandler
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
+		// RemoveRemoteTargetHandler
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 
-			// Remote Tier management operations
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
-			// Tier stats
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
+		// Remote Tier management operations
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
+		// Tier stats
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
 
-			// Cluster Replication APIs
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
+		// Cluster Replication APIs
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
 
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
-		}
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
 
 		if globalIsDistErasure {
 			// Top locks

cmd/auth-handler.go

@@ -218,7 +218,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 	}
 
 	// If AuthZPlugin is set, return without any further checks.
-	if globalAuthZPlugin != nil {
+	if newGlobalAuthZPluginFn() != nil {
 		return claims.Map(), nil
 	}
 

cmd/bitrot.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -27,6 +26,7 @@ import (
 	"io"
 
 	"github.com/minio/highwayhash"
+	"github.com/minio/minio/internal/hash/sha256"
 	"golang.org/x/crypto/blake2b"
 
 	xioutil "github.com/minio/minio/internal/ioutil"

cmd/bucket-handlers.go

@@ -527,7 +527,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			VersionSuspended: vc.Suspended(),
 		}
 
-		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
+		if replicateDeletes || hasLockEnabled || !globalTierConfigMgr.Empty() {
 			if !globalTierConfigMgr.Empty() && object.VersionID == "" && opts.VersionSuspended {
 				opts.VersionID = nullVersionID
 			}
@@ -556,7 +556,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				object.ReplicateDecisionStr = dsc.String()
 			}
 		}
-		if object.VersionID != "" && hasLockEnabled {
+		if hasLockEnabled {
 			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
 				apiErr := errorCodes.ToAPIErr(apiErrCode)
 				deleteResults[index].errInfo = DeleteError{
@@ -1364,7 +1364,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1611,7 +1611,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1624,10 +1624,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if globalSiteReplicationSys.isEnabled() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
-		return
-	}
+
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return
@@ -1639,7 +1636,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return

cmd/bucket-lifecycle_test.go

@@ -203,7 +203,7 @@ func TestObjectIsRemote(t *testing.T) {
 		if got := fi.IsRemote(); got != tc.remote {
 			t.Fatalf("Test %d.a: expected %v got %v", i+1, tc.remote, got)
 		}
-		oi := fi.ToObjectInfo("bucket", "object")
+		oi := fi.ToObjectInfo("bucket", "object", false)
 		if got := oi.IsRemote(); got != tc.remote {
 			t.Fatalf("Test %d.b: expected %v got %v", i+1, tc.remote, got)
 		}

cmd/bucket-metadata-sys.go

@@ -127,20 +127,12 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 		meta.QuotaConfigJSON = configData
 		meta.QuotaConfigUpdatedAt = UTCNow()
 	case objectLockConfig:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
-		}
 		meta.ObjectLockConfigXML = configData
 		meta.ObjectLockConfigUpdatedAt = UTCNow()
 	case bucketVersioningConfig:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
-		}
 		meta.VersioningConfigXML = configData
+		meta.VersioningConfigUpdatedAt = UTCNow()
 	case bucketReplicationConfig:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
-		}
 		meta.ReplicationConfigXML = configData
 		meta.ReplicationConfigUpdatedAt = UTCNow()
 	case bucketTargetsFile:
@@ -193,12 +185,15 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 
 // GetVersioningConfig returns configured versioning config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
+func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, time.Time, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
-		return &versioning.Versioning{}, err
+		if errors.Is(err, errConfigNotFound) {
+			return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, meta.Created, nil
+		}
+		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, time.Time{}, err
 	}
-	return meta.versioningConfig, nil
+	return meta.versioningConfig, meta.VersioningConfigUpdatedAt, nil
 }
 
 // GetTaggingConfig returns configured tagging config
@@ -315,26 +310,27 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
+func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerFn()
 		if objAPI == nil {
-			return nil, errServerNotInitialized
+			return nil, time.Time{}, errServerNotInitialized
 		}
-		return objAPI.GetBucketPolicy(GlobalContext, bucket)
+		p, err := objAPI.GetBucketPolicy(GlobalContext, bucket)
+		return p, UTCNow(), err
 	}
 
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketPolicyNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.policyConfig == nil {
-		return nil, BucketPolicyNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
 	}
-	return meta.policyConfig, nil
+	return meta.policyConfig, meta.PolicyConfigUpdatedAt, nil
 }
 
 // GetQuotaConfig returns configured bucket quota
@@ -369,6 +365,9 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketRemoteTargetNotFound{Bucket: bucket}
+		}
 		return nil, err
 	}
 	if meta.bucketTargetConfig == nil {
@@ -377,20 +376,6 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	return meta.bucketTargetConfig, nil
 }
 
-// GetBucketTarget returns the target for the bucket and arn.
-func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
-	targets, err := sys.GetBucketTargetsConfig(bucket)
-	if err != nil {
-		return madmin.BucketTarget{}, err
-	}
-	for _, t := range targets.Targets {
-		if t.Arn == arn {
-			return t, nil
-		}
-	}
-	return madmin.BucketTarget{}, errConfigNotFound
-}
-
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (BucketMetadata, error) {

cmd/bucket-metadata.go

@@ -87,6 +87,7 @@ type BucketMetadata struct {
 	TaggingConfigUpdatedAt      time.Time
 	QuotaConfigUpdatedAt        time.Time
 	ReplicationConfigUpdatedAt  time.Time
+	VersioningConfigUpdatedAt   time.Time
 
 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.Policy
@@ -364,6 +365,7 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.PolicyConfigUpdatedAt.IsZero() {
 		b.PolicyConfigUpdatedAt = b.Created
 	}
+
 	if b.EncryptionConfigUpdatedAt.IsZero() {
 		b.EncryptionConfigUpdatedAt = b.Created
 	}
@@ -383,6 +385,10 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.ReplicationConfigUpdatedAt.IsZero() {
 		b.ReplicationConfigUpdatedAt = b.Created
 	}
+
+	if b.VersioningConfigUpdatedAt.IsZero() {
+		b.VersioningConfigUpdatedAt = b.Created
+	}
 }
 
 // Save config to supplied ObjectLayer api.

cmd/bucket-metadata_gen.go

@@ -144,6 +144,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
 				return
 			}
+		case "VersioningConfigUpdatedAt":
+			z.VersioningConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -157,9 +163,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 20
+	// map header, size 21
 	// write "Name"
-	err = en.Append(0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -358,15 +364,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
 		return
 	}
+	// write "VersioningConfigUpdatedAt"
+	err = en.Append(0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.VersioningConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 20
+	// map header, size 21
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -425,6 +441,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ReplicationConfigUpdatedAt"
 	o = append(o, 0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.ReplicationConfigUpdatedAt)
+	// string "VersioningConfigUpdatedAt"
+	o = append(o, 0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.VersioningConfigUpdatedAt)
 	return
 }
 
@@ -566,6 +585,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
 				return
 			}
+		case "VersioningConfigUpdatedAt":
+			z.VersioningConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -580,6 +605,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize
 	return
 }

cmd/bucket-policy.go

@@ -38,7 +38,8 @@ type PolicySys struct{}
 
 // Get returns stored bucket policy
 func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
-	return globalBucketMetadataSys.GetPolicyConfig(bucket)
+	policy, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
+	return policy, err
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.

cmd/bucket-replication.go

@@ -87,7 +87,7 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
-func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, APIError) {
+func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, checkRemote bool) (bool, APIError) {
 	var arns []string
 	if rCfg.RoleArn != "" {
 		arns = append(arns, rCfg.RoleArn)
@@ -96,26 +96,29 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 			arns = append(arns, rule.Destination.String())
 		}
 	}
+	var sameTarget bool
 	for _, arnStr := range arns {
 		arn, err := madmin.ParseARN(arnStr)
 		if err != nil {
-			return false, errorCodes.ToAPIErrWithErr(ErrBucketRemoteArnInvalid, err)
+			return sameTarget, errorCodes.ToAPIErrWithErr(ErrBucketRemoteArnInvalid, err)
 		}
 		if arn.Type != madmin.ReplicationService {
-			return false, toAPIError(ctx, BucketRemoteArnTypeInvalid{Bucket: bucket})
+			return sameTarget, toAPIError(ctx, BucketRemoteArnTypeInvalid{Bucket: bucket})
 		}
 		clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arnStr)
 		if clnt == nil {
-			return false, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
+			return sameTarget, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
 		}
-		if found, err := clnt.BucketExists(ctx, arn.Bucket); !found {
-			return false, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, err)
-		}
-		if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
-			if ret.LockEnabled {
-				lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, arn.Bucket)
-				if err != nil || lock != "Enabled" {
-					return false, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, err)
+		if checkRemote { // validate remote bucket
+			if found, err := clnt.BucketExists(ctx, arn.Bucket); !found {
+				return sameTarget, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, err)
+			}
+			if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
+				if ret.LockEnabled {
+					lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, arn.Bucket)
+					if err != nil || lock != "Enabled" {
+						return sameTarget, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, err)
+					}
 				}
 			}
 		}
@@ -123,12 +126,19 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		c, ok := globalBucketTargetSys.arnRemotesMap[arnStr]
 		if ok {
 			if c.EndpointURL().String() == clnt.EndpointURL().String() {
-				sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
-				return sameTarget, toAPIError(ctx, nil)
+				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+				if !sameTarget {
+					sameTarget = selfTarget
+				}
+				continue
 			}
 		}
 	}
-	return false, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
+
+	if len(arns) == 0 {
+		return false, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
+	}
+	return sameTarget, toAPIError(ctx, nil)
 }
 
 type mustReplicateOptions struct {
@@ -180,11 +190,17 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica
 		return
 	}
 
+	// object layer not initialized we return with no decision.
+	if newObjectLayerFn() == nil {
+		return
+	}
+
 	// Disable server-side replication on object prefixes which are excluded
 	// from versioning via the MinIO bucket versioning extension.
 	if globalBucketVersioningSys.PrefixSuspended(bucket, object) {
 		return
 	}
+
 	replStatus := mopts.ReplicationStatus()
 	if replStatus == replication.Replica && !mopts.isMetadataReplication() {
 		return
@@ -2177,10 +2193,6 @@ func (p *ReplicationPool) initResync(ctx context.Context, buckets []BucketInfo,
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
-	// replication applies only to erasure coded setups
-	if !globalIsErasure {
-		return nil
-	}
 	// Load bucket metadata sys in background
 	go p.loadResync(ctx, buckets, objAPI)
 	return nil

cmd/bucket-targets.go

@@ -120,9 +120,6 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 	}
 	if tgt.Type == madmin.ReplicationService {
-		if !globalIsErasure {
-			return NotImplemented{Message: "Replication is not implemented in " + getMinioMode()}
-		}
 		if !globalBucketVersioningSys.Enabled(bucket) {
 			return BucketReplicationSourceNotVersioned{Bucket: bucket}
 		}
@@ -184,9 +181,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	if globalIsGateway {
 		return nil
 	}
-	if !globalIsErasure {
-		return NotImplemented{Message: "Replication is not implemented in " + getMinioMode()}
-	}
 
 	if arnStr == "" {
 		return BucketRemoteArnInvalid{Bucket: bucket}
@@ -201,7 +195,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
 		if err == nil {
-			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{}) {
+			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{OpType: replication.AllReplicationType}) {
 				if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) {
 					sys.RLock()
 					_, ok := sys.arnRemotesMap[arnStr]

cmd/bucket-versioning.go

@@ -17,16 +17,21 @@
 
 package cmd
 
-import "github.com/minio/minio/internal/bucket/versioning"
+import (
+	"strings"
+
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/logger"
+)
 
 // BucketVersioningSys - policy subsystem.
 type BucketVersioningSys struct{}
 
 // Enabled enabled versioning?
 func (sys *BucketVersioningSys) Enabled(bucket string) bool {
-	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	vc, err := sys.Get(bucket)
 	if err != nil {
-		return false
+		logger.CriticalIf(GlobalContext, err)
 	}
 	return vc.Enabled()
 }
@@ -35,18 +40,18 @@ func (sys *BucketVersioningSys) Enabled(bucket string) bool {
 // the given prefix doesn't match any excluded prefixes pattern. This is
 // part of a MinIO versioning configuration extension.
 func (sys *BucketVersioningSys) PrefixEnabled(bucket, prefix string) bool {
-	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	vc, err := sys.Get(bucket)
 	if err != nil {
-		return false
+		logger.CriticalIf(GlobalContext, err)
 	}
 	return vc.PrefixEnabled(prefix)
 }
 
 // Suspended suspended versioning?
 func (sys *BucketVersioningSys) Suspended(bucket string) bool {
-	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	vc, err := sys.Get(bucket)
 	if err != nil {
-		return false
+		logger.CriticalIf(GlobalContext, err)
 	}
 	return vc.Suspended()
 }
@@ -54,9 +59,9 @@ func (sys *BucketVersioningSys) Suspended(bucket string) bool {
 // PrefixSuspended returns true if the given prefix matches an excluded prefix
 // pattern. This is part of a MinIO versioning configuration extension.
 func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
-	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	vc, err := sys.Get(bucket)
 	if err != nil {
-		return false
+		logger.CriticalIf(GlobalContext, err)
 	}
 
 	return vc.PrefixSuspended(prefix)
@@ -64,12 +69,17 @@ func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
 
 // Get returns stored bucket policy
 func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
-	return globalBucketMetadataSys.GetVersioningConfig(bucket)
-}
+	if globalIsGateway {
+		// Gateway does not implement versioning.
+		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
+	}
 
-// Reset BucketVersioningSys to initial state.
-func (sys *BucketVersioningSys) Reset() {
-	// There is currently no internal state.
+	if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) {
+		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
+	}
+
+	vcfg, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	return vcfg, err
 }
 
 // NewBucketVersioningSys - creates new versioning system.

cmd/config-current.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/config/heal"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
@@ -56,6 +57,7 @@ func initHelp() {
 		config.IdentityLDAPSubSys:   xldap.DefaultKVS,
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
 		config.IdentityTLSSubSys:    xtls.DefaultKVS,
+		config.IdentityPluginSubSys: idplugin.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
 		config.PolicyPluginSubSys:   polplugin.DefaultKVS,
 		config.SiteSubSys:           config.DefaultSiteKVS,
@@ -108,6 +110,10 @@ func initHelp() {
 			Key:         config.IdentityTLSSubSys,
 			Description: "enable X.509 TLS certificate SSO support",
 		},
+		config.HelpKV{
+			Key:         config.IdentityPluginSubSys,
+			Description: "enable Identity Plugin via external hook",
+		},
 		config.HelpKV{
 			Key:         config.PolicyPluginSubSys,
 			Description: "enable Access Management Plugin for policy enforcement",
@@ -220,6 +226,7 @@ func initHelp() {
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.IdentityTLSSubSys:    xtls.Help,
+		config.IdentityPluginSubSys: idplugin.Help,
 		config.PolicyOPASubSys:      opa.Help,
 		config.PolicyPluginSubSys:   polplugin.Help,
 		config.LoggerWebhookSubSys:  logger.Help,
@@ -342,6 +349,11 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 		if _, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default]); err != nil {
 			return err
 		}
+	case config.IdentityPluginSubSys:
+		if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
+			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
+			return err
+		}
 	case config.SubnetSubSys:
 		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default]); err != nil {
 			return err
@@ -541,6 +553,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
 
+	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
+		globalRootCAs)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
+	}
+
+	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
+	}
+	globalAuthNPlugin = idplugin.New(authNPluginCfg)
+
 	authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
@@ -558,13 +583,8 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
 		}
 	}
-	globalAuthZPlugin = polplugin.New(authZPluginCfg)
 
-	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
-		globalRootCAs)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
-	}
+	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
 
 	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
 	if err != nil {

cmd/data-scanner.go

@@ -1057,9 +1057,13 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 	fivs = fivs[:lim+1]
 
 	rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
+	vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
+
+	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
+
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
-		obj := fi.ToObjectInfo(i.bucket, i.objectPath())
+		obj := fi.ToObjectInfo(i.bucket, i.objectPath(), versioned)
 		// skip versions with object locking enabled
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			if i.debug {

cmd/endpoint.go

@@ -582,7 +582,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 			return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
 		endpoints = append(endpoints, endpoint)
-		setupType = FSSetupType
+		setupType = ErasureSDSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {

cmd/endpoint_test.go

@@ -231,10 +231,10 @@ func TestCreateEndpoints(t *testing.T) {
 	}{
 		{"localhost", [][]string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")},
 
-		// FS Setup
+		// Erasure Single Drive
 		{"localhost:9000", [][]string{{"http://localhost/d1"}}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
-		{":443", [][]string{{"/d1"}}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, FSSetupType, nil},
-		{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, FSSetupType, nil},
+		{":443", [][]string{{"/d1"}}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
+		{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 
 		// Erasure Setup with PathEndpointType

cmd/erasure-coding.go

@@ -41,7 +41,7 @@ type Erasure struct {
 // NewErasure creates a new ErasureStorage.
 func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) {
 	// Check the parameters for sanity now.
-	if dataBlocks <= 0 || parityBlocks <= 0 {
+	if dataBlocks <= 0 || parityBlocks < 0 {
 		return e, reedsolomon.ErrInvShardNum
 	}
 

cmd/erasure-healing.go

@@ -418,10 +418,19 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		return result, nil
 	}
 
+	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+		// When disk to heal count is greater than parity blocks we should simply error out.
+		err := fmt.Errorf("more disks are expected to heal than parity, returned errors: %v -> %s/%s(%s)", errs, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	cleanFileInfo := func(fi FileInfo) FileInfo {
-		// Returns a copy of the 'fi' with checksums and parts nil'ed.
+		// Returns a copy of the 'fi' with erasure index, checksums and inline data niled.
 		nfi := fi
-		if !fi.IsRemote() {
+		if !nfi.IsRemote() {
+			nfi.Data = nil
 			nfi.Erasure.Index = 0
 			nfi.Erasure.Checksums = nil
 		}
@@ -432,12 +441,25 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	tmpID := mustGetUUID()
 	migrateDataDir := mustGetUUID()
 
+	// Reorder so that we have data disks first and parity disks next.
+	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
+	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
+
 	copyPartsMetadata := make([]FileInfo, len(partsMetadata))
+	for i := range latestDisks {
+		if latestDisks[i] == nil {
+			continue
+		}
+		copyPartsMetadata[i] = partsMetadata[i]
+	}
+
 	for i := range outDatedDisks {
 		if outDatedDisks[i] == nil {
 			continue
 		}
-		copyPartsMetadata[i] = partsMetadata[i]
+		// Make sure to write the FileInfo information
+		// that is expected to be in quorum.
 		partsMetadata[i] = cleanFileInfo(latestMeta)
 	}
 
@@ -456,12 +478,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 	}
 
-	// Reorder so that we have data disks first and parity disks next.
-	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
-	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
-	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
-	copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
-
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
 		// Heal each part. erasureHealFile() will write the healed
 		// part to .minio/tmp/uuid/ which needs to be renamed later to
@@ -486,7 +502,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 				checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber)
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
-				readers[i] = newBitrotReader(disk, partsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
+				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
 					checksumInfo.Hash, erasure.ShardSize())
 			}
 			writers := make([]io.Writer, len(outDatedDisks))

cmd/erasure-metadata-utils.go

@@ -178,10 +178,6 @@ func shuffleDisksAndPartsMetadataByIndex(disks []StorageAPI, metaArr []FileInfo,
 			inconsistent++
 			continue
 		}
-		if len(fi.Data) != len(meta.Data) {
-			inconsistent++
-			continue
-		}
 		if meta.XLV1 != fi.XLV1 {
 			inconsistent++
 			continue
@@ -229,12 +225,6 @@ func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo,
 			// if object was ever written previously.
 			continue
 		}
-		if !init && len(fi.Data) != len(partsMetadata[index].Data) {
-			// Check for length of data parts only when
-			// fi.ModTime is not empty - ModTime is always set,
-			// if object was ever written previously.
-			continue
-		}
 		if !init && fi.XLV1 != partsMetadata[index].XLV1 {
 			continue
 		}

cmd/erasure-metadata.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"net/http"
@@ -28,6 +27,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -99,15 +99,15 @@ func (fi FileInfo) IsValid() bool {
 		fi.Erasure.Index <= dataBlocks+parityBlocks &&
 		len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks))
 	return ((dataBlocks >= parityBlocks) &&
-		(dataBlocks != 0) && (parityBlocks != 0) &&
+		(dataBlocks > 0) && (parityBlocks >= 0) &&
 		correctIndexes)
 }
 
 // ToObjectInfo - Converts metadata to object info.
-func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
+func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInfo {
 	object = decodeDirObject(object)
 	versionID := fi.VersionID
-	if (globalBucketVersioningSys.PrefixEnabled(bucket, object) || globalBucketVersioningSys.PrefixSuspended(bucket, object)) && versionID == "" {
+	if versioned && versionID == "" {
 		versionID = nullVersionID
 	}
 
@@ -284,7 +284,7 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 
 func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (FileInfo, error) {
 	// with less quorum return error.
-	if quorum < 2 {
+	if quorum < 1 {
 		return FileInfo{}, errErasureReadQuorum
 	}
 	metaHashes := make([]string, len(metaArr))
@@ -297,8 +297,6 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 				fmt.Fprintf(h, "part.%d", part.Number)
 			}
 			fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
-			// make sure that length of Data is same
-			fmt.Fprintf(h, "%v", len(meta.Data))
 
 			// ILM transition fields
 			fmt.Fprint(h, meta.TransitionStatus)
@@ -400,6 +398,10 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 // readQuorum is the min required disks to read data.
 // writeQuorum is the min required disks to write data.
 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
+	if defaultParityCount == 0 {
+		return 1, 1, nil
+	}
+
 	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
 	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
 	if err != nil {

cmd/erasure-multipart.go

@@ -1027,7 +1027,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	fi.IsLatest = true
 
 	// Success, return object info.
-	return fi.ToObjectInfo(bucket, object), nil
+	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
 // AbortMultipartUpload - aborts an ongoing multipart operation

cmd/erasure-object.go

@@ -117,7 +117,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		if srcOpts.VersionID == "" {
 			return oi, toObjectErr(errFileNotFound, srcBucket, srcObject)
 		}
-		return fi.ToObjectInfo(srcBucket, srcObject), toObjectErr(errMethodNotAllowed, srcBucket, srcObject)
+		return fi.ToObjectInfo(srcBucket, srcObject, srcOpts.Versioned || srcOpts.VersionSuspended), toObjectErr(errMethodNotAllowed, srcBucket, srcObject)
 	}
 
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
@@ -133,15 +133,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		modTime = UTCNow()
 	}
 
-	// If the data is not inlined, we may end up incorrectly
-	// inlining the data here, that leads to an inconsistent
-	// situation where some objects are were not inlined
-	// were now inlined, make sure to `nil` the Data such
-	// that xl.meta is written as expected.
-	if !fi.InlineData() {
-		fi.Data = nil
-	}
-
 	fi.VersionID = versionID // set any new versionID we might have created
 	fi.ModTime = modTime     // set modTime for the new versionID
 	if !dstOpts.MTime.IsZero() {
@@ -151,12 +142,15 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	fi.Metadata = srcInfo.UserDefined
 	srcInfo.UserDefined["etag"] = srcInfo.ETag
 
+	inlineData := fi.InlineData()
+	freeVersionID := fi.TierFreeVersionID()
+	freeVersionMarker := fi.TierFreeVersion()
+
 	// Update `xl.meta` content on each disks.
 	for index := range metaArr {
 		if metaArr[index].IsValid() {
 			metaArr[index].ModTime = modTime
 			metaArr[index].VersionID = versionID
-			metaArr[index].Metadata = srcInfo.UserDefined
 			if !metaArr[index].InlineData() {
 				// If the data is not inlined, we may end up incorrectly
 				// inlining the data here, that leads to an inconsistent
@@ -165,6 +159,17 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 				// that xl.meta is written as expected.
 				metaArr[index].Data = nil
 			}
+			metaArr[index].Metadata = srcInfo.UserDefined
+			// Preserve existing values
+			if inlineData {
+				metaArr[index].SetInlineData()
+			}
+			if freeVersionID != "" {
+				metaArr[index].SetTierFreeVersionID(freeVersionID)
+			}
+			if freeVersionMarker {
+				metaArr[index].SetTierFreeVersion()
+			}
 		}
 	}
 
@@ -173,7 +178,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
-	return fi.ToObjectInfo(srcBucket, srcObject), nil
+	return fi.ToObjectInfo(srcBucket, srcObject, srcOpts.Versioned || srcOpts.VersionSuspended), nil
 }
 
 // GetObjectNInfo - returns object info and an object
@@ -240,7 +245,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		}
 	}
 
-	objInfo := fi.ToObjectInfo(bucket, object)
+	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if objInfo.DeleteMarker {
 		if opts.VersionID == "" {
 			return &GetObjectReader{
@@ -504,23 +509,45 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 	}
 
 	readQuorum := (len(disks) + 1) / 2
-	merged := mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)
+	meta := &xlMetaV2{versions: mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)}
+	lfi, err := meta.ToFileInfo(bucket, object, "")
+	if err != nil {
+		for i := range errs {
+			if errs[i] == nil {
+				errs[i] = err
+			}
+		}
+		return metaFileInfos, errs
+	}
+	if !lfi.IsValid() {
+		for i := range errs {
+			if errs[i] == nil {
+				errs[i] = errCorruptedFormat
+			}
+		}
+		return metaFileInfos, errs
+	}
+
 	for index := range metadataArray {
 		if metadataArray[index] == nil {
 			continue
 		}
 
-		metadataArray[index].versions = merged
-
 		// make sure to preserve this for diskmtime based healing bugfix.
 		diskMTime := metaFileInfos[index].DiskMTime
 		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, "")
-		if errs[index] == nil {
+		if errs[index] != nil {
+			continue
+		}
+
+		if metaFileInfos[index].IsValid() && metaFileInfos[index].ModTime.Equal(lfi.ModTime) {
 			versionID := metaFileInfos[index].VersionID
 			if versionID == "" {
 				versionID = nullVersionID
 			}
-			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
+			if readData {
+				metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
+			}
 			metaFileInfos[index].DiskMTime = diskMTime
 		}
 	}
@@ -588,6 +615,8 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		if metaArr[i].IsValid() && metaArr[i].ModTime.Equal(fi.ModTime) {
 			continue
 		}
+		metaArr[i] = FileInfo{}
+		onlineDisks[i] = nil
 		missingBlocks++
 	}
 
@@ -609,7 +638,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 	if err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
-	objInfo = fi.ToObjectInfo(bucket, object)
+	objInfo = fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if fi.Deleted {
 		if opts.VersionID == "" || opts.DeleteMarker {
 			return objInfo, toObjectErr(errFileNotFound, bucket, object)
@@ -633,7 +662,7 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 		wquorum++
 	}
 
-	objInfo = fi.ToObjectInfo(bucket, object)
+	objInfo = fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if !fi.VersionPurgeStatus().Empty() && opts.VersionID != "" {
 		// Make sure to return object info to provide extra information.
 		return objInfo, wquorum, toObjectErr(errMethodNotAllowed, bucket, object)
@@ -825,7 +854,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key)
 	}
 
-	return fi.ToObjectInfo(minioMetaBucket, key), nil
+	return fi.ToObjectInfo(minioMetaBucket, key, opts.Versioned || opts.VersionSuspended), nil
 }
 
 // PutObject - creates an object upon reading from the input stream
@@ -1130,7 +1159,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// we are adding a new version to this object under the namespace lock, so this is the latest version.
 	fi.IsLatest = true
 
-	return fi.ToObjectInfo(bucket, object), nil
+	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
 func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo, forceDelMarker bool) error {
@@ -1322,12 +1351,17 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	disks := er.getDisks()
 	g := errgroup.WithNErrs(len(disks))
+	dirPrefix := encodeDirObject(prefix)
 	for index := range disks {
 		index := index
 		g.Go(func() error {
 			if disks[index] == nil {
 				return nil
 			}
+			// Deletes
+			// - The prefix and its children
+			// - The prefix__XLDIR__
+			defer disks[index].Delete(ctx, bucket, dirPrefix, true)
 			return disks[index].Delete(ctx, bucket, prefix, true)
 		}, index)
 	}
@@ -1499,7 +1533,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
 				return objInfo, toObjectErr(err, bucket, object)
 			}
-			return fi.ToObjectInfo(bucket, object), nil
+			return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 		}
 	}
 
@@ -1604,7 +1638,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 		opts.VersionID = fi.VersionID
 	}
 
-	objInfo := fi.ToObjectInfo(bucket, object)
+	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if opts.EvalMetadataFn != nil {
 		if err := opts.EvalMetadataFn(objInfo); err != nil {
 			return ObjectInfo{}, err
@@ -1620,7 +1654,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
-	return fi.ToObjectInfo(bucket, object), nil
+	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
 // PutObjectTags - replace or add tags to an existing object
@@ -1684,7 +1718,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
-	return fi.ToObjectInfo(bucket, object), nil
+	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
 // updateObjectMeta will update the metadata of a file.
@@ -1822,7 +1856,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		break
 	}
 
-	objInfo := fi.ToObjectInfo(bucket, object)
+	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	sendEvent(eventArgs{
 		EventName:  eventName,
 		BucketName: bucket,
@@ -1877,7 +1911,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 	}
 
-	oi = actualfi.ToObjectInfo(bucket, object)
+	oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
 	if len(oi.Parts) == 1 {
 		var rs *HTTPRangeSpec

cmd/erasure-object_test.go

@@ -20,11 +20,14 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	crand "crypto/rand"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"testing"
 
@@ -1048,3 +1051,157 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		})
 	}
 }
+
+// In some deployments, one object has data inlined in one disk and not inlined in other disks.
+func TestGetObjectInlineNotInline(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create a backend with 4 disks named disk{1...4}, this name convention
+	// because we will unzip some object data from a sample archive.
+	const numDisks = 4
+	path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var fsDirs []string
+	for i := 1; i <= numDisks; i++ {
+		fsDirs = append(fsDirs, filepath.Join(path, fmt.Sprintf("disk%d", i)))
+	}
+
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		removeRoots(fsDirs)
+		t.Fatal(err)
+	}
+
+	// cleaning up of temporary test directories
+	defer objLayer.Shutdown(context.Background())
+	defer removeRoots(fsDirs)
+
+	// Create a testbucket
+	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", BucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unzip sample object data to the existing disks
+	err = unzipArchive("testdata/xl-meta-inline-notinline.zip", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Try to read the object and check its md5sum
+	gr, err := objLayer.GetObjectNInfo(ctx, "testbucket", "file", nil, nil, readLock, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
+	}
+
+	h := md5.New()
+	_, err = io.Copy(h, gr)
+	if err != nil {
+		t.Fatalf("Expected GetObject reading data to succeed, but failed with %v", err)
+	}
+	gr.Close()
+
+	const expectedHash = "fffb6377948ebea75ad2b8058e849ef5"
+	foundHash := fmt.Sprintf("%x", h.Sum(nil))
+	if foundHash != expectedHash {
+		t.Fatalf("Expected data to have md5sum = `%s`, found `%s`", expectedHash, foundHash)
+	}
+}
+
+// Test reading an object with some outdated data in some disks
+func TestGetObjectWithOutdatedDisks(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create an instance of xl backend.
+	obj, fsDirs, err := prepareErasure(ctx, 6)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Cleanup backend directories.
+	defer obj.Shutdown(context.Background())
+	defer removeRoots(fsDirs)
+
+	z := obj.(*erasureServerPools)
+	sets := z.serverPools[0]
+	xl := sets.sets[0]
+
+	origErasureDisks := xl.getDisks()
+
+	testCases := []struct {
+		bucket    string
+		versioned bool
+		object    string
+		content   []byte
+	}{
+		{"bucket1", false, "object1", []byte("aaaaaaaaaaaaaaaa")},
+		{"bucket2", false, "object2", bytes.Repeat([]byte{'a'}, smallFileThreshold*2)},
+		{"bucket3", true, "version1", []byte("aaaaaaaaaaaaaaaa")},
+		{"bucket4", true, "version2", bytes.Repeat([]byte{'a'}, smallFileThreshold*2)},
+	}
+
+	for i, testCase := range testCases {
+		// Step 1: create a bucket
+		err = z.MakeBucketWithLocation(ctx, testCase.bucket, BucketOptions{VersioningEnabled: testCase.versioned})
+		if err != nil {
+			t.Fatalf("Test %d: Failed to create a bucket: %v", i+1, err)
+		}
+
+		// Step 2: Upload an object with a random content
+		initialData := bytes.Repeat([]byte{'b'}, len(testCase.content))
+		sets.erasureDisksMu.Lock()
+		xl.getDisks = func() []StorageAPI { return origErasureDisks }
+		sets.erasureDisksMu.Unlock()
+		_, err = z.PutObject(ctx, testCase.bucket, testCase.object, mustGetPutObjReader(t, bytes.NewReader(initialData), int64(len(initialData)), "", ""),
+			ObjectOptions{Versioned: testCase.versioned})
+		if err != nil {
+			t.Fatalf("Test %d: Failed to upload a random object: %v", i+1, err)
+		}
+
+		// Step 3: Upload the object with some disks offline
+		sets.erasureDisksMu.Lock()
+		xl.getDisks = func() []StorageAPI {
+			disks := make([]StorageAPI, len(origErasureDisks))
+			copy(disks, origErasureDisks)
+			disks[0] = nil
+			disks[1] = nil
+			return disks
+		}
+		sets.erasureDisksMu.Unlock()
+		_, err = z.PutObject(ctx, testCase.bucket, testCase.object, mustGetPutObjReader(t, bytes.NewReader(testCase.content), int64(len(testCase.content)), "", ""),
+			ObjectOptions{Versioned: testCase.versioned})
+		if err != nil {
+			t.Fatalf("Test %d: Failed to upload the final object: %v", i+1, err)
+		}
+
+		// Step 4: Try to read the object back and check its md5sum
+		sets.erasureDisksMu.Lock()
+		xl.getDisks = func() []StorageAPI { return origErasureDisks }
+		sets.erasureDisksMu.Unlock()
+		gr, err := z.GetObjectNInfo(ctx, testCase.bucket, testCase.object, nil, nil, readLock, ObjectOptions{})
+		if err != nil {
+			t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
+		}
+
+		h := md5.New()
+		h.Write(testCase.content)
+		expectedHash := h.Sum(nil)
+
+		h.Reset()
+		_, err = io.Copy(h, gr)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to calculate md5sum of the object: %v", i+1, err)
+		}
+		gr.Close()
+		foundHash := h.Sum(nil)
+
+		if !bytes.Equal(foundHash, expectedHash) {
+			t.Fatalf("Test %d: Expected data to have md5sum = `%x`, found `%x`", i+1, expectedHash, foundHash)
+		}
+	}
+}

cmd/erasure-server-pool.go

@@ -61,6 +61,22 @@ func (z *erasureServerPools) SinglePool() bool {
 
 // Initialize new pool of erasure sets.
 func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
+	if endpointServerPools.NEndpoints() == 1 {
+		ep := endpointServerPools[0]
+		storageDisks, format, err := waitForFormatErasure(true, ep.Endpoints, 1, ep.SetCount, ep.DrivesPerSet, "", "")
+		if err != nil {
+			return nil, err
+		}
+
+		objLayer, err := newErasureSingle(ctx, storageDisks[0], format)
+		if err != nil {
+			return nil, err
+		}
+
+		globalLocalDrives = storageDisks
+		return objLayer, nil
+	}
+
 	var (
 		deploymentID       string
 		distributionAlgo   string
@@ -233,8 +249,9 @@ func (z *erasureServerPools) SetDriveCounts() []int {
 type serverPoolsAvailableSpace []poolAvailableSpace
 
 type poolAvailableSpace struct {
-	Index     int
-	Available uint64
+	Index      int
+	Available  uint64
+	MaxUsedPct int // Used disk percentage of most filled disk, rounded down.
 }
 
 // TotalAvailable - total available space
@@ -246,10 +263,41 @@ func (p serverPoolsAvailableSpace) TotalAvailable() uint64 {
 	return total
 }
 
+// FilterMaxUsed will filter out any pools that has used percent bigger than max,
+// unless all have that, in which case all are preserved.
+func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
+	// We aren't modifying p, only entries in it, so we don't need to receive a pointer.
+	if len(p) <= 1 {
+		// Nothing to do.
+		return
+	}
+	var ok bool
+	for _, z := range p {
+		if z.MaxUsedPct < max {
+			ok = true
+			break
+		}
+	}
+	if !ok {
+		// All above limit.
+		// Do not modify
+		return
+	}
+
+	// Remove entries that are above.
+	for i, z := range p {
+		if z.MaxUsedPct < max {
+			continue
+		}
+		p[i].Available = 0
+	}
+}
+
 // getAvailablePoolIdx will return an index that can hold size bytes.
 // -1 is returned if no serverPools have available space for the size given.
 func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, object string, size int64) int {
 	serverPools := z.getServerPoolsAvailableSpace(ctx, bucket, object, size)
+	serverPools.FilterMaxUsed(100 - (100 * diskReserveFraction))
 	total := serverPools.TotalAvailable()
 	if total == 0 {
 		return -1
@@ -270,11 +318,13 @@ func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, ob
 
 // getServerPoolsAvailableSpace will return the available space of each pool after storing the content.
 // If there is not enough space the pool will return 0 bytes available.
+// The size of each will be multiplied by the number of sets.
 // Negative sizes are seen as 0 bytes.
 func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, bucket, object string, size int64) serverPoolsAvailableSpace {
 	serverPools := make(serverPoolsAvailableSpace, len(z.serverPools))
 
 	storageInfos := make([][]*DiskInfo, len(z.serverPools))
+	nSets := make([]int, len(z.serverPools))
 	g := errgroup.WithNErrs(len(z.serverPools))
 	for index := range z.serverPools {
 		index := index
@@ -282,9 +332,11 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 		if z.IsSuspended(index) {
 			continue
 		}
+		pool := z.serverPools[index]
+		nSets[index] = pool.setCount
 		g.Go(func() error {
 			// Get the set where it would be placed.
-			storageInfos[index] = getDiskInfos(ctx, z.serverPools[index].getHashedSet(object).getDisks())
+			storageInfos[index] = getDiskInfos(ctx, pool.getHashedSet(object).getDisks()...)
 			return nil
 		}, index)
 	}
@@ -298,15 +350,30 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 			serverPools[i] = poolAvailableSpace{Index: i}
 			continue
 		}
+		var maxUsedPct int
 		for _, disk := range zinfo {
-			if disk == nil {
+			if disk == nil || disk.Total == 0 {
 				continue
 			}
 			available += disk.Total - disk.Used
+
+			// set maxUsedPct to the value from the disk with the least space percentage.
+			if pctUsed := int(disk.Used * 100 / disk.Total); pctUsed > maxUsedPct {
+				maxUsedPct = pctUsed
+			}
 		}
+
+		// Since we are comparing pools that may have a different number of sets
+		// we multiply by the number of sets in the pool.
+		// This will compensate for differences in set sizes
+		// when choosing destination pool.
+		// Different set sizes are already compensated by less disks.
+		available *= uint64(nSets[i])
+
 		serverPools[i] = poolAvailableSpace{
-			Index:     i,
-			Available: available,
+			Index:      i,
+			Available:  available,
+			MaxUsedPct: maxUsedPct,
 		}
 	}
 	return serverPools
@@ -882,7 +949,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 	object = encodeDirObject(object)
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()), data.Size()) {
+		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size()) {
 			return ObjectInfo{}, toObjectErr(errDiskFull)
 		}
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
@@ -1274,7 +1341,7 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()), -1) {
+		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1) {
 			return "", toObjectErr(errDiskFull)
 		}
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
@@ -1689,6 +1756,8 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 		defer cancel()
 		defer close(results)
 
+		versioned := opts.Versioned || opts.VersionSuspended
+
 		for _, erasureSet := range z.serverPools {
 			var wg sync.WaitGroup
 			for _, set := range erasureSet.sets {
@@ -1716,12 +1785,12 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						if opts.WalkAscending {
 							for i := len(fivs.Versions) - 1; i >= 0; i-- {
 								version := fivs.Versions[i]
-								results <- version.ToObjectInfo(bucket, version.Name)
+								results <- version.ToObjectInfo(bucket, version.Name, versioned)
 							}
 							return
 						}
 						for _, version := range fivs.Versions {
-							results <- version.ToObjectInfo(bucket, version.Name)
+							results <- version.ToObjectInfo(bucket, version.Name, versioned)
 						}
 					}
 

cmd/erasure-sets.go

@@ -1252,7 +1252,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {
-			closeStorageDisks(storageDisks)
+			closeStorageDisks(storageDisks...)
 		}
 	}(storageDisks)
 

cmd/erasure.go

@@ -79,7 +79,7 @@ func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker {
 // Shutdown function for object storage interface.
 func (er erasureObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
-	closeStorageDisks(er.getDisks())
+	closeStorageDisks(er.getDisks()...)
 	return nil
 }
 

cmd/format-erasure.go

@@ -40,6 +40,9 @@ const (
 	// Represents Erasure backend.
 	formatBackendErasure = "xl"
 
+	// Represents Erasure backend - single drive
+	formatBackendErasureSingle = "xl-single"
+
 	// formatErasureV1.Erasure.Version - version '1'.
 	formatErasureVersionV1 = "1"
 
@@ -146,6 +149,9 @@ func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 {
 	format := &formatErasureV3{}
 	format.Version = formatMetaVersionV1
 	format.Format = formatBackendErasure
+	if setLen == 1 {
+		format.Format = formatBackendErasureSingle
+	}
 	format.ID = mustGetUUID()
 	format.Erasure.Version = formatErasureVersionV3
 	format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgoV3
@@ -170,8 +176,8 @@ func formatGetBackendErasureVersion(b []byte) (string, error) {
 	if meta.Version != formatMetaVersionV1 {
 		return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
 	}
-	if meta.Format != formatBackendErasure {
-		return "", fmt.Errorf(`found backend type %s, expected %s`, meta.Format, formatBackendErasure)
+	if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle {
+		return "", fmt.Errorf(`found backend type %s, expected %s or %s`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
 	}
 	// Erasure backend found, proceed to detect version.
 	format := &formatErasureVersionDetect{}
@@ -291,7 +297,7 @@ func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, er
 func countErrs(errs []error, err error) int {
 	i := 0
 	for _, err1 := range errs {
-		if err1 == err {
+		if err1 == err || errors.Is(err1, err) {
 			i++
 		}
 	}
@@ -410,7 +416,7 @@ func checkFormatErasureValue(formatErasure *formatErasureV3, disk StorageAPI) er
 	if formatErasure.Version != formatMetaVersionV1 {
 		return fmt.Errorf("Unsupported version of backend format [%s] found on %s", formatErasure.Version, disk)
 	}
-	if formatErasure.Format != formatBackendErasure {
+	if formatErasure.Format != formatBackendErasure && formatErasure.Format != formatBackendErasureSingle {
 		return fmt.Errorf("Unsupported backend format [%s] found on %s", formatErasure.Format, disk)
 	}
 	if formatErasure.Erasure.Version != formatErasureVersionV3 {
@@ -643,7 +649,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 }
 
 // relinquishes the underlying connection for all storage disks.
-func closeStorageDisks(storageDisks []StorageAPI) {
+func closeStorageDisks(storageDisks ...StorageAPI) {
 	var wg sync.WaitGroup
 	for _, disk := range storageDisks {
 		if disk == nil {

cmd/format-fs.go

@@ -240,6 +240,9 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 		if err != nil {
 			return nil, err
 		}
+		if formatBackend == formatBackendErasureSingle {
+			return nil, errFreshDisk
+		}
 		if formatBackend != formatBackendFS {
 			return nil, fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
 		}
@@ -319,6 +322,10 @@ func formatFSFixDeploymentID(ctx context.Context, fsFormatPath string) error {
 		rlk.Close()
 		return err
 	}
+	if formatBackend == formatBackendErasureSingle {
+		rlk.Close()
+		return errFreshDisk
+	}
 	if formatBackend != formatBackendFS {
 		rlk.Close()
 		return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)

cmd/fs-v1-metadata_test.go

@@ -45,6 +45,8 @@ func TestFSV1MetadataObjInfo(t *testing.T) {
 
 // TestReadFSMetadata - readFSMetadata testing with a healthy and faulty disk
 func TestReadFSMetadata(t *testing.T) {
+	t.Skip()
+
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
 
@@ -80,6 +82,7 @@ func TestReadFSMetadata(t *testing.T) {
 
 // TestWriteFSMetadata - tests of writeFSMetadata with healthy disk.
 func TestWriteFSMetadata(t *testing.T) {
+	t.Skip()
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
 

cmd/fs-v1-multipart_test.go

@@ -32,6 +32,7 @@ import (
 
 // Tests cleanup multipart uploads for filesystem backend.
 func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
+	t.Skip()
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
@@ -88,6 +89,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 
 // TestNewMultipartUploadFaultyDisk - test NewMultipartUpload with faulty disks
 func TestNewMultipartUploadFaultyDisk(t *testing.T) {
+	t.Skip()
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)

cmd/fs-v1.go

@@ -142,6 +142,11 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, config.ErrUnableToWriteInBackend(err).Hint(hint)
 	}
 
+	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
+	if _, err = fsStat(ctx, fsFormatPath); err != nil && os.IsNotExist(err) {
+		return nil, errFreshDisk
+	}
+
 	// Assign a new UUID for FS minio mode. Each server instance
 	// gets its own UUID for temporary file transaction.
 	fsUUID := mustGetUUID()

cmd/fs-v1_test.go

@@ -51,6 +51,8 @@ func TestNewFS(t *testing.T) {
 // TestFSShutdown - initialize a new FS object layer then calls
 // Shutdown to check returned results
 func TestFSShutdown(t *testing.T) {
+	t.Skip()
+
 	bucketName := "testbucket"
 	objectName := "object"
 	// Create and return an fsObject with its path in the disk
@@ -83,6 +85,8 @@ func TestFSShutdown(t *testing.T) {
 
 // TestFSGetBucketInfo - test GetBucketInfo with healty and faulty disks
 func TestFSGetBucketInfo(t *testing.T) {
+	t.Skip()
+
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
@@ -165,6 +169,7 @@ func TestFSPutObject(t *testing.T) {
 
 // TestFSDeleteObject - test fs.DeleteObject() with healthy and corrupted disks
 func TestFSDeleteObject(t *testing.T) {
+	t.Skip()
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
@@ -209,6 +214,7 @@ func TestFSDeleteObject(t *testing.T) {
 
 // TestFSDeleteBucket - tests for fs DeleteBucket
 func TestFSDeleteBucket(t *testing.T) {
+	t.Skip()
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
@@ -249,6 +255,7 @@ func TestFSDeleteBucket(t *testing.T) {
 
 // TestFSListBuckets - tests for fs ListBuckets
 func TestFSListBuckets(t *testing.T) {
+	t.Skip()
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)

cmd/gateway-main.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -32,6 +33,7 @@ import (
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/config"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -292,6 +294,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		SecretKey: globalActiveCred.SecretKey,
 	})
 	if err != nil {
+		if errors.Is(err, errFreshDisk) {
+			err = config.ErrInvalidFSValue(err)
+		}
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 	newObject = NewGatewayLayerWithLocker(newObject)

cmd/global-heal.go

@@ -340,7 +340,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 // healObject heals given object path in deep to fix bitrot.
 func healObject(bucket, object, versionID string, scan madmin.HealScanMode) {
 	// Get background heal sequence to send elements to heal
+	globalHealStateLK.Lock()
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	globalHealStateLK.Unlock()
 	if ok {
 		bgSeq.queueHealTask(healSource{
 			bucket:    bucket,

cmd/globals.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
@@ -73,6 +74,7 @@ const (
 	globalWindowsOSName            = "windows"
 	globalMacOSName                = "darwin"
 	globalMinioModeFS              = "mode-server-fs"
+	globalMinioModeErasureSD       = "mode-server-xl-single"
 	globalMinioModeErasure         = "mode-server-xl"
 	globalMinioModeDistErasure     = "mode-server-distributed-xl"
 	globalMinioModeGatewayPrefix   = "mode-gateway-"
@@ -113,6 +115,10 @@ const (
 	// diskFillFraction is the fraction of a disk we allow to be filled.
 	diskFillFraction = 0.99
 
+	// diskReserveFraction is the fraction of a disk where we will fill other server pools first.
+	// If all pools reach this, we will use all pools with regular placement.
+	diskReserveFraction = 0.15
+
 	// diskAssumeUnknownSize is the size to assume when an unknown size upload is requested.
 	diskAssumeUnknownSize = 1 << 30
 
@@ -136,6 +142,9 @@ var (
 	// Indicates if the running minio server is an erasure-code backend.
 	globalIsErasure = false
 
+	// Indicates if the running minio server is in single drive XL mode.
+	globalIsErasureSD = false
+
 	// Indicates if the running minio is in gateway mode.
 	globalIsGateway = false
 
@@ -196,10 +205,13 @@ var (
 	globalAPIConfig = apiConfig{listQuorum: "strict"}
 
 	globalStorageClass storageclass.Config
+
 	globalLDAPConfig   xldap.Config
 	globalOpenIDConfig openid.Config
 	globalSTSTLSConfig xtls.Config
 
+	globalAuthNPlugin *idplugin.AuthNPlugin
+
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
 
@@ -355,4 +367,18 @@ var (
 	// Add new variable global values here.
 )
 
+var globalAuthZPluginMutex sync.Mutex
+
+func newGlobalAuthZPluginFn() *polplugin.AuthZPlugin {
+	globalAuthZPluginMutex.Lock()
+	defer globalAuthZPluginMutex.Unlock()
+	return globalAuthZPlugin
+}
+
+func setGlobalAuthZPlugin(authz *polplugin.AuthZPlugin) {
+	globalAuthZPluginMutex.Lock()
+	globalAuthZPlugin = authz
+	globalAuthZPluginMutex.Unlock()
+}
+
 var errSelfTestFailure = errors.New("self test failed. unsafe to start server")

cmd/handler-api.go

@@ -116,10 +116,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
 		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
 		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
-
-		if globalIsErasure {
-			logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
-		}
+		logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {

cmd/hasher.go

@@ -19,8 +19,9 @@ package cmd
 
 import (
 	"crypto/md5"
-	"crypto/sha256"
 	"encoding/hex"
+
+	"github.com/minio/minio/internal/hash/sha256"
 )
 
 // getSHA256Hash returns SHA-256 hash in hex encoding of given data.

cmd/iam-store.go

@@ -1795,14 +1795,29 @@ func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string
 	cache := store.rlock()
 	defer store.runlock()
 
+	userExists := false
 	var tempAccounts []auth.Credentials
 	for _, v := range cache.iamUsersMap {
-		if v.IsTemp() && v.ParentUser == accessKey {
-			// Hide secret key & session key here
-			v.SecretKey = ""
-			v.SessionToken = ""
-			tempAccounts = append(tempAccounts, v)
+		isDerived := false
+		if v.IsServiceAccount() || v.IsTemp() {
+			isDerived = true
 		}
+
+		if !isDerived && v.AccessKey == accessKey {
+			userExists = true
+		} else if isDerived && v.ParentUser == accessKey {
+			userExists = true
+			if v.IsTemp() {
+				// Hide secret key & session key here
+				v.SecretKey = ""
+				v.SessionToken = ""
+				tempAccounts = append(tempAccounts, v)
+			}
+		}
+	}
+
+	if !userExists {
+		return nil, errNoSuchUser
 	}
 
 	return tempAccounts, nil
@@ -1813,14 +1828,29 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 	cache := store.rlock()
 	defer store.runlock()
 
+	userExists := false
 	var serviceAccounts []auth.Credentials
 	for _, v := range cache.iamUsersMap {
-		if v.IsServiceAccount() && v.ParentUser == accessKey {
-			// Hide secret key & session key here
-			v.SecretKey = ""
-			v.SessionToken = ""
-			serviceAccounts = append(serviceAccounts, v)
+		isDerived := false
+		if v.IsServiceAccount() || v.IsTemp() {
+			isDerived = true
 		}
+
+		if !isDerived && v.AccessKey == accessKey {
+			userExists = true
+		} else if isDerived && v.ParentUser == accessKey {
+			userExists = true
+			if v.IsServiceAccount() {
+				// Hide secret key & session key here
+				v.SecretKey = ""
+				v.SessionToken = ""
+				serviceAccounts = append(serviceAccounts, v)
+			}
+		}
+	}
+
+	if !userExists {
+		return nil, errNoSuchUser
 	}
 
 	return serviceAccounts, nil

cmd/iam.go

@@ -38,6 +38,8 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	xldap "github.com/minio/minio/internal/config/identity/ldap"
+	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -80,6 +82,8 @@ type IAMSys struct {
 	sync.Mutex
 
 	iamRefreshInterval time.Duration
+	ldapConfig         xldap.Config  // only valid if usersSysType is LDAPUsers
+	openIDConfig       openid.Config // only valid if OpenID is configured
 
 	usersSysType UsersSysType
 
@@ -164,7 +168,7 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 
 // initStore initializes IAM stores
 func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
-	if globalLDAPConfig.Enabled {
+	if sys.ldapConfig.Enabled {
 		sys.EnableLDAPSys()
 	}
 
@@ -224,6 +228,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	sys.Lock()
 	defer sys.Unlock()
 
+	sys.ldapConfig = globalLDAPConfig.Clone()
+	sys.openIDConfig = globalOpenIDConfig.Clone()
 	sys.iamRefreshInterval = iamRefreshInterval
 
 	// Initialize IAM store
@@ -308,26 +314,28 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	refreshInterval := sys.iamRefreshInterval
+
 	// Set up polling for expired accounts and credentials purging.
 	switch {
-	case globalOpenIDConfig.ProviderEnabled():
+	case sys.openIDConfig.ProviderEnabled():
 		go func() {
-			timer := time.NewTimer(sys.iamRefreshInterval)
+			timer := time.NewTimer(refreshInterval)
 			defer timer.Stop()
 			for {
 				select {
 				case <-timer.C:
 					sys.purgeExpiredCredentialsForExternalSSO(ctx)
 
-					timer.Reset(sys.iamRefreshInterval)
+					timer.Reset(refreshInterval)
 				case <-ctx.Done():
 					return
 				}
 			}
 		}()
-	case globalLDAPConfig.Enabled:
+	case sys.ldapConfig.Enabled:
 		go func() {
-			timer := time.NewTimer(sys.iamRefreshInterval)
+			timer := time.NewTimer(refreshInterval)
 			defer timer.Stop()
 
 			for {
@@ -336,7 +344,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 					sys.purgeExpiredCredentialsForLDAP(ctx)
 					sys.updateGroupMembershipsForLDAP(ctx)
 
-					timer.Reset(sys.iamRefreshInterval)
+					timer.Reset(refreshInterval)
 				case <-ctx.Done():
 					return
 				}
@@ -347,20 +355,18 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	// Start watching changes to storage.
 	go sys.watch(ctx)
 
-	// Load RoleARN
-	if rolePolicyMap := globalOpenIDConfig.GetRoleInfo(); rolePolicyMap != nil {
-		// Validate that policies associated with roles are defined.
-		for _, rolePolicies := range rolePolicyMap {
-			ps := newMappedPolicy(rolePolicies).toSlice()
-			numPolicies := len(ps)
-			validPolicies, _ := sys.store.FilterPolicies(rolePolicies, "")
-			numValidPolicies := len(strings.Split(validPolicies, ","))
-			if numPolicies != numValidPolicies {
-				logger.LogIf(ctx, fmt.Errorf("Some specified role policies (in %s) were not defined - role policies will not be enabled.", rolePolicies))
-				return
-			}
-		}
-		sys.rolesMap = rolePolicyMap
+	// Load RoleARNs
+	sys.rolesMap = make(map[arn.ARN]string)
+
+	// From OpenID
+	if riMap := globalOpenIDConfig.GetRoleInfo(); riMap != nil {
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
+	// From AuthN plugin if enabled.
+	if globalAuthNPlugin != nil {
+		riMap := globalAuthNPlugin.GetRoleInfo()
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
 	}
 
 	sys.printIAMRoles()
@@ -369,6 +375,32 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	logger.Info("Finished loading IAM sub-system (took %.1fs of %.1fs to load data).", now.Sub(iamLoadStart).Seconds(), now.Sub(iamInitStart).Seconds())
 }
 
+func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[arn.ARN]string) {
+	// Validate that policies associated with roles are defined. If
+	// authZ plugin is set, role policies are just claims sent to
+	// the plugin and they need not exist.
+	//
+	// If some mapped policies do not exist, we print some error
+	// messages but continue any way - they can be fixed in the
+	// running server by creating the policies after start up.
+	for arn, rolePolicies := range m {
+		specifiedPoliciesSet := newMappedPolicy(rolePolicies).policySet()
+		validPolicies, _ := sys.store.FilterPolicies(rolePolicies, "")
+		knownPoliciesSet := newMappedPolicy(validPolicies).policySet()
+		unknownPoliciesSet := specifiedPoliciesSet.Difference(knownPoliciesSet)
+		if len(unknownPoliciesSet) > 0 {
+			if globalAuthZPlugin == nil {
+				// Print a warning that some policies mapped to a role are not defined.
+				errMsg := fmt.Errorf(
+					"The policies \"%s\" mapped to role ARN %s are not defined - this role may not work as expected.",
+					unknownPoliciesSet.ToSlice(), arn.String())
+				logger.LogIf(ctx, errMsg)
+			}
+		}
+		sys.rolesMap[arn] = rolePolicies
+	}
+}
+
 // Prints IAM role ARNs.
 func (sys *IAMSys) printIAMRoles() {
 	if len(sys.rolesMap) == 0 {
@@ -687,7 +719,7 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 		return errServerNotInitialized
 	}
 
-	if globalAuthZPlugin != nil {
+	if newGlobalAuthZPluginFn() != nil {
 		// If OPA is set, we do not need to set a policy mapping.
 		policyName = ""
 	}
@@ -1138,7 +1170,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 			continue
 		}
 		roleArn = roleArns[0]
-		u, err := globalOpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
+		u, err := sys.openIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue
@@ -1160,14 +1192,14 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var allDistNames []string
 	for parentUser := range parentUsers {
-		if !globalLDAPConfig.IsLDAPUserDN(parentUser) {
+		if !sys.ldapConfig.IsLDAPUserDN(parentUser) {
 			continue
 		}
 
 		allDistNames = append(allDistNames, parentUser)
 	}
 
-	expiredUsers, err := globalLDAPConfig.GetNonEligibleUserDistNames(allDistNames)
+	expiredUsers, err := sys.ldapConfig.GetNonEligibleUserDistNames(allDistNames)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1189,7 +1221,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// DN to ldap username mapping for each LDAP user
 	parentUserToLDAPUsernameMap := make(map[string]string)
 	for _, cred := range allCreds {
-		if !globalLDAPConfig.IsLDAPUserDN(cred.ParentUser) {
+		if !sys.ldapConfig.IsLDAPUserDN(cred.ParentUser) {
 			continue
 		}
 		// Check if this is the first time we are
@@ -1237,7 +1269,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	}
 
 	// 2. Query LDAP server for groups of the LDAP users collected.
-	updatedGroups, err := globalLDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	updatedGroups, err := sys.ldapConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1660,8 +1692,8 @@ func (sys *IAMSys) GetCombinedPolicy(policies ...string) iampolicy.Policy {
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	// If opa is configured, use OPA always.
-	if globalAuthZPlugin != nil {
-		ok, err := globalAuthZPlugin.IsAllowed(args)
+	if authz := newGlobalAuthZPluginFn(); authz != nil {
+		ok, err := authz.IsAllowed(args)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 		}

cmd/lock-rest-server.go

@@ -209,22 +209,7 @@ func (l *lockRESTServer) ForceUnlockHandler(w http.ResponseWriter, r *http.Reque
 // lockMaintenance loops over all locks and discards locks
 // that have not been refreshed for some time.
 func lockMaintenance(ctx context.Context) {
-	// Wait until the object API is ready
-	// no need to start the lock maintenance
-	// if ObjectAPI is not initialized.
-
-	var objAPI ObjectLayer
-
-	for {
-		objAPI = newObjectLayerFn()
-		if objAPI == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		break
-	}
-
-	if _, ok := objAPI.(*erasureServerPools); !ok {
+	if !globalIsDistErasure {
 		return
 	}
 

cmd/metacache-bucket.go

@@ -51,10 +51,12 @@ func newBucketMetacache(bucket string, cleanup bool) *bucketMetacache {
 	if cleanup {
 		// Recursively delete all caches.
 		objAPI := newObjectLayerFn()
-		ez, ok := objAPI.(*erasureServerPools)
-		if ok {
-			ctx := context.Background()
-			ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator))
+		if objAPI != nil {
+			ez, ok := objAPI.(renameAllStorager)
+			if ok {
+				ctx := context.Background()
+				ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator))
+			}
 		}
 	}
 	return &bucketMetacache{
@@ -207,9 +209,15 @@ func (b *bucketMetacache) cloneCaches() (map[string]metacache, map[string][]stri
 // Deletes are performed concurrently.
 func (b *bucketMetacache) deleteAll() {
 	ctx := context.Background()
-	ez, ok := newObjectLayerFn().(*erasureServerPools)
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return
+	}
+
+	ez, ok := objAPI.(renameAllStorager)
 	if !ok {
-		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be *erasurePools"))
+		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be 'renameAllStorager'"))
 		return
 	}
 

cmd/metacache-entries.go

@@ -168,8 +168,10 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
 	if !isXL2V1Format(e.metadata) {
 		return false
 	}
-	if meta, _ := isIndexedMetaV2(e.metadata); meta != nil {
+	if meta, _, err := isIndexedMetaV2(e.metadata); meta != nil {
 		return meta.IsLatestDeleteMarker()
+	} else if err != nil {
+		return true
 	}
 	// Fall back...
 	xlMeta, err := e.xlmeta()
@@ -438,6 +440,9 @@ func (m metaCacheEntriesSorted) shallowClone() metaCacheEntriesSorted {
 func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, afterV string) (versions []ObjectInfo) {
 	versions = make([]ObjectInfo, 0, m.len())
 	prevPrefix := ""
+	vcfg, _ := globalBucketVersioningSys.Get(bucket)
+	versioned := vcfg != nil && vcfg.Versioned(prefix)
+
 	for _, entry := range m.o {
 		if entry.isObject() {
 			if delimiter != "" {
@@ -473,7 +478,7 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 			}
 
 			for _, version := range fiVersions {
-				versions = append(versions, version.ToObjectInfo(bucket, entry.name))
+				versions = append(versions, version.ToObjectInfo(bucket, entry.name, versioned))
 			}
 
 			continue
@@ -509,6 +514,10 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (objects []ObjectInfo) {
 	objects = make([]ObjectInfo, 0, m.len())
 	prevPrefix := ""
+
+	vcfg, _ := globalBucketVersioningSys.Get(bucket)
+	versioned := vcfg != nil && vcfg.Versioned(prefix)
+
 	for _, entry := range m.o {
 		if entry.isObject() {
 			if delimiter != "" {
@@ -531,7 +540,7 @@ func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (ob
 
 			fi, err := entry.fileInfo(bucket)
 			if err == nil {
-				objects = append(objects, fi.ToObjectInfo(bucket, entry.name))
+				objects = append(objects, fi.ToObjectInfo(bucket, entry.name, versioned))
 			}
 			continue
 		}

cmd/metacache-manager.go

@@ -56,7 +56,7 @@ func (m *metacacheManager) initManager() {
 			objAPI = newObjectLayerFn()
 		}
 
-		if !globalIsErasure {
+		if globalIsGateway {
 			return
 		}
 

cmd/metacache-server-pool.go

@@ -259,6 +259,286 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	return entries, nil
 }
 
+// listPath will return the requested entries.
+// If no more entries are in the listing io.EOF is returned,
+// otherwise nil or an unexpected error is returned.
+// The listPathOptions given will be checked and modified internally.
+// Required important fields are Bucket, Prefix, Separator.
+// Other important fields are Limit, Marker.
+// List ID always derived from the Marker.
+func (es *erasureSingle) listPath(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, es); err != nil {
+		return entries, err
+	}
+
+	// Marker is set validate pre-condition.
+	if o.Marker != "" && o.Prefix != "" {
+		// Marker not common with prefix is not implemented. Send an empty response
+		if !HasPrefix(o.Marker, o.Prefix) {
+			return entries, io.EOF
+		}
+	}
+
+	// With max keys of zero we have reached eof, return right here.
+	if o.Limit == 0 {
+		return entries, io.EOF
+	}
+
+	// For delimiter and prefix as '/' we do not list anything at all
+	// along // with the prefix. On a flat namespace with 'prefix'
+	// as '/' we don't have any entries, since all the keys are
+	// of form 'keyName/...'
+	if strings.HasPrefix(o.Prefix, SlashSeparator) {
+		return entries, io.EOF
+	}
+
+	// If delimiter is slashSeparator we must return directories of
+	// the non-recursive scan unless explicitly requested.
+	o.IncludeDirectories = o.Separator == slashSeparator
+	if (o.Separator == slashSeparator || o.Separator == "") && !o.Recursive {
+		o.Recursive = o.Separator != slashSeparator
+		o.Separator = slashSeparator
+	} else {
+		// Default is recursive, if delimiter is set then list non recursive.
+		o.Recursive = true
+	}
+
+	// Decode and get the optional list id from the marker.
+	o.parseMarker()
+	o.BaseDir = baseDirFromPrefix(o.Prefix)
+	o.Transient = o.Transient || isReservedOrInvalidBucket(o.Bucket, false)
+	o.SetFilter()
+	if o.Transient {
+		o.Create = false
+	}
+
+	// We have 2 cases:
+	// 1) Cold listing, just list.
+	// 2) Returning, but with no id. Start async listing.
+	// 3) Returning, with ID, stream from list.
+	//
+	// If we don't have a list id we must ask the server if it has a cache or create a new.
+	if o.ID != "" && !o.Transient {
+		// Create or ping with handout...
+		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+		var c *metacache
+		if rpc == nil {
+			resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
+			c = &resp
+		} else {
+			c, err = rpc.GetMetacacheListing(ctx, *o)
+		}
+		if err != nil {
+			if errors.Is(err, context.Canceled) {
+				// Context is canceled, return at once.
+				// request canceled, no entries to return
+				return entries, io.EOF
+			}
+			if !errors.Is(err, context.DeadlineExceeded) {
+				o.debugln("listPath: got error", err)
+			}
+			o.Transient = true
+			o.Create = false
+			o.ID = mustGetUUID()
+		} else {
+			if c.fileNotFound {
+				// No cache found, no entries found.
+				return entries, io.EOF
+			}
+			if c.status == scanStateError || c.status == scanStateNone {
+				o.ID = ""
+				o.Create = false
+				o.debugln("scan status", c.status, " - waiting a roundtrip to create")
+			} else {
+				// Continue listing
+				o.ID = c.id
+				go func(meta metacache) {
+					// Continuously update while we wait.
+					t := time.NewTicker(metacacheMaxClientWait / 10)
+					defer t.Stop()
+					select {
+					case <-ctx.Done():
+						// Request is done, stop updating.
+						return
+					case <-t.C:
+						meta.lastHandout = time.Now()
+						if rpc == nil {
+							meta, _ = localMetacacheMgr.updateCacheEntry(meta)
+						}
+						meta, _ = rpc.UpdateMetacacheListing(ctx, meta)
+					}
+				}(*c)
+			}
+		}
+
+		// We have an existing list ID, continue streaming.
+		if o.Create {
+			o.debugln("Creating", o)
+			entries, err = es.listAndSave(ctx, o)
+			if err == nil || err == io.EOF {
+				return entries, err
+			}
+			entries.truncate(0)
+		} else {
+			o.debugln("Resuming", o)
+			entries, err = es.streamMetadataParts(ctx, *o)
+			entries.reuse = true // We read from stream and are not sharing results.
+			if err == nil {
+				return entries, nil
+			}
+		}
+		if IsErr(err, []error{
+			nil,
+			context.Canceled,
+			context.DeadlineExceeded,
+			// io.EOF is expected and should be returned but no need to log it.
+			io.EOF,
+		}...) {
+			// Expected good errors we don't need to return error.
+			return entries, err
+		}
+		entries.truncate(0)
+		go func() {
+			rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
+			if rpc != nil {
+				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
+				defer cancel()
+				c, err := rpc.GetMetacacheListing(ctx, *o)
+				if err == nil {
+					c.error = "no longer used"
+					c.status = scanStateError
+					rpc.UpdateMetacacheListing(ctx, *c)
+				}
+			}
+		}()
+		o.ID = ""
+
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Resuming listing from drives failed %w, proceeding to do raw listing", err))
+		}
+	}
+
+	// Do listing in-place.
+	// Create output for our results.
+	// Create filter for results.
+	o.debugln("Raw List", o)
+	filterCh := make(chan metaCacheEntry, o.Limit)
+	listCtx, cancelList := context.WithCancel(ctx)
+	filteredResults := o.gatherResults(listCtx, filterCh)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	var listErr error
+
+	go func(o listPathOptions) {
+		defer wg.Done()
+		o.Limit = 0
+		listErr = es.listMerged(listCtx, o, filterCh)
+		o.debugln("listMerged returned with", listErr)
+	}(*o)
+
+	entries, err = filteredResults()
+	cancelList()
+	wg.Wait()
+	if listErr != nil && !errors.Is(listErr, context.Canceled) {
+		return entries, listErr
+	}
+	entries.reuse = true
+	truncated := entries.len() > o.Limit || err == nil
+	entries.truncate(o.Limit)
+	if !o.Transient && truncated {
+		if o.ID == "" {
+			entries.listID = mustGetUUID()
+		} else {
+			entries.listID = o.ID
+		}
+	}
+	if !truncated {
+		return entries, io.EOF
+	}
+	return entries, nil
+}
+
+// listMerged will list across all sets and return a merged results stream.
+// The result channel is closed when no more results are expected.
+func (es *erasureSingle) listMerged(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) error {
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	var listErr error
+	var inputs []chan metaCacheEntry
+
+	innerResults := make(chan metaCacheEntry, 100)
+	inputs = append(inputs, innerResults)
+
+	mu.Lock()
+	listCtx, cancelList := context.WithCancel(ctx)
+	defer cancelList()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err := es.listPathInner(listCtx, o, innerResults)
+		mu.Lock()
+		defer mu.Unlock()
+		listErr = err
+	}()
+	mu.Unlock()
+
+	// Do lifecycle filtering.
+	if o.Lifecycle != nil {
+		filterIn := make(chan metaCacheEntry, 10)
+		go filterLifeCycle(ctx, o.Bucket, *o.Lifecycle, o.Retention, filterIn, results)
+		// Replace results.
+		results = filterIn
+	}
+
+	// Gather results to a single channel.
+	err := mergeEntryChannels(ctx, inputs, results, func(existing, other *metaCacheEntry) (replace bool) {
+		// Pick object over directory
+		if existing.isDir() && !other.isDir() {
+			return true
+		}
+		if !existing.isDir() && other.isDir() {
+			return false
+		}
+		eMeta, err := existing.xlmeta()
+		if err != nil {
+			return true
+		}
+		oMeta, err := other.xlmeta()
+		if err != nil {
+			return false
+		}
+		// Replace if modtime is newer
+		if !oMeta.latestModtime().Equal(oMeta.latestModtime()) {
+			return oMeta.latestModtime().After(eMeta.latestModtime())
+		}
+		// Use NumVersions as a final tiebreaker.
+		return len(oMeta.versions) > len(eMeta.versions)
+	})
+
+	cancelList()
+	wg.Wait()
+	if err != nil {
+		return err
+	}
+	if listErr != nil {
+		if contextCanceled(ctx) {
+			return nil
+		}
+		if listErr.Error() == io.EOF.Error() {
+			return nil
+		}
+		logger.LogIf(ctx, listErr)
+		return listErr
+	}
+	if contextCanceled(ctx) {
+		return ctx.Err()
+	}
+	return nil
+}
+
 // listMerged will list across all sets and return a merged results stream.
 // The result channel is closed when no more results are expected.
 func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) error {
@@ -362,6 +642,8 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 // function closes 'out' and exits.
 func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle, lr lock.Retention, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
 	defer close(out)
+
+	vcfg, _ := globalBucketVersioningSys.Get(bucket)
 	for {
 		var obj metaCacheEntry
 		var ok bool
@@ -378,7 +660,10 @@ func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle,
 		if err != nil {
 			continue
 		}
-		objInfo := fi.ToObjectInfo(bucket, obj.name)
+
+		versioned := vcfg != nil && vcfg.Versioned(obj.name)
+
+		objInfo := fi.ToObjectInfo(bucket, obj.name, versioned)
 		action := evalActionFromLifecycle(ctx, lc, lr, objInfo, false)
 		switch action {
 		case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
@@ -395,6 +680,73 @@ func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle,
 	}
 }
 
+func (es *erasureSingle) listAndSave(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	// Use ID as the object name...
+	o.pool = 0
+	o.set = 0
+	saver := es
+
+	// Disconnect from call above, but cancel on exit.
+	listCtx, cancel := context.WithCancel(GlobalContext)
+	saveCh := make(chan metaCacheEntry, metacacheBlockSize)
+	inCh := make(chan metaCacheEntry, metacacheBlockSize)
+	outCh := make(chan metaCacheEntry, o.Limit)
+
+	filteredResults := o.gatherResults(ctx, outCh)
+
+	mc := o.newMetacache()
+	meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)), o: *o}
+
+	// Save listing...
+	go func() {
+		if err := saver.saveMetaCacheStream(listCtx, &meta, saveCh); err != nil {
+			meta.setErr(err.Error())
+		}
+		cancel()
+	}()
+
+	// Do listing...
+	go func(o listPathOptions) {
+		err := es.listMerged(listCtx, o, inCh)
+		if err != nil {
+			meta.setErr(err.Error())
+		}
+		o.debugln("listAndSave: listing", o.ID, "finished with ", err)
+	}(*o)
+
+	// Keep track of when we return since we no longer have to send entries to output.
+	var funcReturned bool
+	var funcReturnedMu sync.Mutex
+	defer func() {
+		funcReturnedMu.Lock()
+		funcReturned = true
+		funcReturnedMu.Unlock()
+	}()
+	// Write listing to results and saver.
+	go func() {
+		var returned bool
+		for entry := range inCh {
+			if !returned {
+				funcReturnedMu.Lock()
+				returned = funcReturned
+				funcReturnedMu.Unlock()
+				outCh <- entry
+				if returned {
+					close(outCh)
+				}
+			}
+			entry.reusable = returned
+			saveCh <- entry
+		}
+		if !returned {
+			close(outCh)
+		}
+		close(saveCh)
+	}()
+
+	return filteredResults()
+}
+
 func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {
 	// Use ID as the object name...
 	o.pool = z.getAvailablePoolIdx(ctx, minioMetaBucket, o.ID, 10<<20)

cmd/metacache-set.go

@@ -543,6 +543,170 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 	}
 }
 
+func (es *erasureSingle) streamMetadataParts(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	retries := 0
+	rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
+
+	for {
+		if contextCanceled(ctx) {
+			return entries, ctx.Err()
+		}
+
+		// If many failures, check the cache state.
+		if retries > 10 {
+			err := o.checkMetacacheState(ctx, rpc)
+			if err != nil {
+				return entries, fmt.Errorf("remote listing canceled: %w", err)
+			}
+			retries = 1
+		}
+
+		const retryDelay = 250 * time.Millisecond
+		// All operations are performed without locks, so we must be careful and allow for failures.
+		// Read metadata associated with the object from a disk.
+		if retries > 0 {
+			_, err := es.disk.ReadVersion(ctx, minioMetaBucket,
+				o.objectPath(0), "", false)
+			if err != nil {
+				time.Sleep(retryDelay)
+				retries++
+				continue
+			}
+		}
+
+		// Load first part metadata...
+		// Read metadata associated with the object from all disks.
+		fi, metaArr, onlineDisks, err := es.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true)
+		if err != nil {
+			switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
+			case ObjectNotFound:
+				retries++
+				time.Sleep(retryDelay)
+				continue
+			case InsufficientReadQuorum:
+				retries++
+				time.Sleep(retryDelay)
+				continue
+			default:
+				return entries, fmt.Errorf("reading first part metadata: %w", err)
+			}
+		}
+
+		partN, err := o.findFirstPart(fi)
+		switch {
+		case err == nil:
+		case errors.Is(err, io.ErrUnexpectedEOF):
+			if retries == 10 {
+				err := o.checkMetacacheState(ctx, rpc)
+				if err != nil {
+					return entries, fmt.Errorf("remote listing canceled: %w", err)
+				}
+				retries = -1
+			}
+			retries++
+			time.Sleep(retryDelay)
+			continue
+		case errors.Is(err, io.EOF):
+			return entries, io.EOF
+		}
+
+		// We got a stream to start at.
+		loadedPart := 0
+		for {
+			if contextCanceled(ctx) {
+				return entries, ctx.Err()
+			}
+
+			if partN != loadedPart {
+				if retries > 10 {
+					err := o.checkMetacacheState(ctx, rpc)
+					if err != nil {
+						return entries, fmt.Errorf("waiting for next part %d: %w", partN, err)
+					}
+					retries = 1
+				}
+
+				if retries > 0 {
+					// Load from one disk only
+					_, err := es.disk.ReadVersion(ctx, minioMetaBucket,
+						o.objectPath(partN), "", false)
+					if err != nil {
+						time.Sleep(retryDelay)
+						retries++
+						continue
+					}
+				}
+
+				// Load partN metadata...
+				fi, metaArr, onlineDisks, err = es.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
+				if err != nil {
+					time.Sleep(retryDelay)
+					retries++
+					continue
+				}
+				loadedPart = partN
+				bi, err := getMetacacheBlockInfo(fi, partN)
+				logger.LogIf(ctx, err)
+				if err == nil {
+					if bi.pastPrefix(o.Prefix) {
+						return entries, io.EOF
+					}
+				}
+			}
+
+			pr, pw := io.Pipe()
+			go func() {
+				werr := es.getObjectWithFileInfo(ctx, minioMetaBucket, o.objectPath(partN), 0,
+					fi.Size, pw, fi, metaArr, onlineDisks)
+				pw.CloseWithError(werr)
+			}()
+
+			tmp := newMetacacheReader(pr)
+			e, err := tmp.filter(o)
+			pr.CloseWithError(err)
+			entries.o = append(entries.o, e.o...)
+			if o.Limit > 0 && entries.len() > o.Limit {
+				entries.truncate(o.Limit)
+				return entries, nil
+			}
+			if err == nil {
+				// We stopped within the listing, we are done for now...
+				return entries, nil
+			}
+			if err != nil && err.Error() != io.EOF.Error() {
+				switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) {
+				case ObjectNotFound:
+					retries++
+					time.Sleep(retryDelay)
+					continue
+				case InsufficientReadQuorum:
+					retries++
+					time.Sleep(retryDelay)
+					continue
+				default:
+					logger.LogIf(ctx, err)
+					return entries, err
+				}
+			}
+
+			// We finished at the end of the block.
+			// And should not expect any more results.
+			bi, err := getMetacacheBlockInfo(fi, partN)
+			logger.LogIf(ctx, err)
+			if err != nil || bi.EOS {
+				// We are done and there are no more parts.
+				return entries, io.EOF
+			}
+			if bi.endedPrefix(o.Prefix) {
+				// Nothing more for prefix.
+				return entries, io.EOF
+			}
+			partN++
+			retries = 0
+		}
+	}
+}
+
 // getListQuorum interprets list quorum values and returns appropriate
 // acceptable quorum expected for list operations
 func getListQuorum(quorum string, driveCount int) int {
@@ -562,6 +726,53 @@ func getListQuorum(quorum string, driveCount int) int {
 	return 3
 }
 
+// Will return io.EOF if continuing would not yield more results.
+func (es *erasureSingle) listPathInner(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
+	defer close(results)
+	o.debugf(color.Green("listPath:")+" with options: %#v", o)
+
+	// How to resolve results.
+	resolver := metadataResolutionParams{
+		dirQuorum: 1,
+		objQuorum: 1,
+		bucket:    o.Bucket,
+	}
+
+	// Maximum versions requested for "latest" object
+	// resolution on versioned buckets, this is to be only
+	// used when o.Versioned is false
+	if !o.Versioned {
+		resolver.requestedVersions = 1
+	}
+
+	ctxDone := ctx.Done()
+	return listPathRaw(ctx, listPathRawOptions{
+		disks:        []StorageAPI{es.disk},
+		bucket:       o.Bucket,
+		path:         o.BaseDir,
+		recursive:    o.Recursive,
+		filterPrefix: o.FilterPrefix,
+		minDisks:     1,
+		forwardTo:    o.Marker,
+		agreed: func(entry metaCacheEntry) {
+			select {
+			case <-ctxDone:
+			case results <- entry:
+			}
+		},
+		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+			// Results Disagree :-(
+			entry, ok := entries.resolve(&resolver)
+			if ok {
+				select {
+				case <-ctxDone:
+				case results <- *entry:
+				}
+			}
+		},
+	})
+}
+
 // Will return io.EOF if continuing would not yield more results.
 func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
 	defer close(results)
@@ -654,6 +865,133 @@ func (m *metaCacheRPC) setErr(err string) {
 	*m.meta = meta
 }
 
+func (es *erasureSingle) saveMetaCacheStream(ctx context.Context, mc *metaCacheRPC, entries <-chan metaCacheEntry) (err error) {
+	o := mc.o
+	o.debugf(color.Green("saveMetaCacheStream:")+" with options: %#v", o)
+
+	metaMu := &mc.mu
+	rpc := mc.rpc
+	cancel := mc.cancel
+	defer func() {
+		o.debugln(color.Green("saveMetaCacheStream:")+"err:", err)
+		if err != nil && !errors.Is(err, io.EOF) {
+			go mc.setErr(err.Error())
+			cancel()
+		}
+	}()
+
+	defer cancel()
+	// Save continuous updates
+	go func() {
+		var err error
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		var exit bool
+		for !exit {
+			select {
+			case <-ticker.C:
+			case <-ctx.Done():
+				exit = true
+			}
+			metaMu.Lock()
+			meta := *mc.meta
+			meta, err = o.updateMetacacheListing(meta, rpc)
+			if err == nil && time.Since(meta.lastHandout) > metacacheMaxClientWait {
+				cancel()
+				exit = true
+				meta.status = scanStateError
+				meta.error = fmt.Sprintf("listing canceled since time since last handout was %v ago", time.Since(meta.lastHandout).Round(time.Second))
+				o.debugln(color.Green("saveMetaCacheStream: ") + meta.error)
+				meta, err = o.updateMetacacheListing(meta, rpc)
+			}
+			if err == nil {
+				*mc.meta = meta
+				if meta.status == scanStateError {
+					cancel()
+					exit = true
+				}
+			}
+			metaMu.Unlock()
+		}
+	}()
+
+	const retryDelay = 200 * time.Millisecond
+	const maxTries = 5
+
+	// Keep destination...
+	// Write results to disk.
+	bw := newMetacacheBlockWriter(entries, func(b *metacacheBlock) error {
+		// if the block is 0 bytes and its a first block skip it.
+		// skip only this for Transient caches.
+		if len(b.data) == 0 && b.n == 0 && o.Transient {
+			return nil
+		}
+		o.debugln(color.Green("saveMetaCacheStream:")+" saving block", b.n, "to", o.objectPath(b.n))
+		r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
+		logger.LogIf(ctx, err)
+		custom := b.headerKV()
+		_, err = es.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
+			UserDefined: custom,
+		})
+		if err != nil {
+			mc.setErr(err.Error())
+			cancel()
+			return err
+		}
+		if b.n == 0 {
+			return nil
+		}
+		// Update block 0 metadata.
+		var retries int
+		for {
+			meta := b.headerKV()
+			fi := FileInfo{
+				Metadata: make(map[string]string, len(meta)),
+			}
+			for k, v := range meta {
+				fi.Metadata[k] = v
+			}
+			err := es.updateObjectMeta(ctx, minioMetaBucket, o.objectPath(0), fi, es.disk)
+			if err == nil {
+				break
+			}
+			switch err.(type) {
+			case ObjectNotFound:
+				return err
+			case StorageErr:
+				return err
+			case InsufficientReadQuorum:
+			default:
+				logger.LogIf(ctx, err)
+			}
+			if retries >= maxTries {
+				return err
+			}
+			retries++
+			time.Sleep(retryDelay)
+		}
+		return nil
+	})
+
+	// Blocks while consuming entries or an error occurs.
+	err = bw.Close()
+	if err != nil {
+		mc.setErr(err.Error())
+	}
+	metaMu.Lock()
+	defer metaMu.Unlock()
+	if mc.meta.error != "" {
+		return err
+	}
+	// Save success
+	mc.meta.status = scanStateSuccess
+	meta, err := o.updateMetacacheListing(*mc.meta, rpc)
+	if err == nil {
+		*mc.meta = meta
+	}
+	return nil
+}
+
 func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCacheRPC, entries <-chan metaCacheEntry) (err error) {
 	o := mc.o
 	o.debugf(color.Green("saveMetaCacheStream:")+" with options: %#v", o)

cmd/metacache.go

@@ -155,9 +155,9 @@ func (m *metacache) delete(ctx context.Context) {
 		logger.LogIf(ctx, errors.New("metacache.delete: no object layer"))
 		return
 	}
-	ez, ok := objAPI.(*erasureServerPools)
+	ez, ok := objAPI.(renameAllStorager)
 	if !ok {
-		logger.LogIf(ctx, errors.New("metacache.delete: expected objAPI to be *erasureServerPools"))
+		logger.LogIf(ctx, errors.New("metacache.delete: expected objAPI to be 'renameAllStorager'"))
 		return
 	}
 	ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(m.bucket, m.id))

cmd/metrics-v2.go

@@ -1365,7 +1365,7 @@ func getMinioHealingMetrics() *MetricsGroup {
 	mg := &MetricsGroup{}
 	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
 		metrics = make([]Metric, 0, 5)
-		if !globalIsErasure {
+		if globalIsGateway {
 			return
 		}
 		bgSeq, exists := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
@@ -1817,7 +1817,7 @@ func getClusterStorageMetrics() *MetricsGroup {
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
 		// Service not initialized yet
-		if objLayer == nil || !globalIsErasure {
+		if objLayer == nil || globalIsGateway {
 			return
 		}
 

cmd/metrics.go

@@ -132,7 +132,7 @@ func nodeHealthMetricsPrometheus(ch chan<- prometheus.Metric) {
 // collects healing specific metrics for MinIO instance in Prometheus specific format
 // and sends to given channel
 func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
-	if !globalIsErasure {
+	if globalIsGateway {
 		return
 	}
 	bgSeq, exists := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)

cmd/object-api-listobjects_test.go

@@ -35,9 +35,6 @@ func TestListObjectsVersionedFolders(t *testing.T) {
 }
 
 func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
-	if instanceType == FSTestStr {
-		return
-	}
 	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
@@ -317,9 +314,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 }
 
 func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, versioned bool) {
-	if instanceType == FSTestStr && versioned {
-		return
-	}
 	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
@@ -1020,10 +1014,6 @@ func TestDeleteObjectVersionMarker(t *testing.T) {
 }
 
 func testDeleteObjectVersion(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
-	if instanceType == FSTestStr {
-		return
-	}
-
 	t, _ := t1.(*testing.T)
 
 	testBuckets := []string{
@@ -1101,10 +1091,6 @@ func TestListObjectVersions(t *testing.T) {
 
 // Unit test for ListObjectVersions
 func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
-	if instanceType == FSTestStr {
-		return
-	}
-
 	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
@@ -1886,16 +1872,14 @@ func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestEr
 
 // Initialize FS backend for the benchmark.
 func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
-	var err error
-	obj, err = NewFSObjectLayer(disk)
+	obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(disk))
 	if err != nil {
-		t.Fatal("Unexpected err: ", err)
+		t.Fatal(err)
 	}
 
 	newTestConfig(globalMinioDefaultRegion, obj)
 
 	initAllSubsystems()
-
 	return obj
 }
 

cmd/object-api-multipart_test.go

@@ -1620,39 +1620,35 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 				t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
 			}
 
-			// ListObjectParts returns empty response always in FS mode
-			if instanceType != FSTestStr {
-				// Asserting IsTruncated.
-				if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
-					t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
-					continue
+			// Asserting IsTruncated.
+			if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
+				t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
+				continue
+			}
+			// Asserting NextPartNumberMarker.
+			if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
+				t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
+				continue
+			}
+			// Asserting the number of Parts.
+			if len(expectedResult.Parts) != len(actualResult.Parts) {
+				t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
+				continue
+			}
+			// Iterating over the partInfos and asserting the fields.
+			for j, actualMetaData := range actualResult.Parts {
+				//  Asserting the PartNumber in the PartInfo.
+				if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
+					t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
 				}
-				// Asserting NextPartNumberMarker.
-				if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
-					t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
-					continue
+				//  Asserting the Size in the PartInfo.
+				if actualMetaData.Size != expectedResult.Parts[j].Size {
+					t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
 				}
-				// Asserting the number of Parts.
-				if len(expectedResult.Parts) != len(actualResult.Parts) {
-					t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
-					continue
+				//  Asserting the ETag in the PartInfo.
+				if actualMetaData.ETag != expectedResult.Parts[j].ETag {
+					t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
 				}
-				// Iterating over the partInfos and asserting the fields.
-				for j, actualMetaData := range actualResult.Parts {
-					//  Asserting the PartNumber in the PartInfo.
-					if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber {
-						t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber)
-					}
-					//  Asserting the Size in the PartInfo.
-					if actualMetaData.Size != expectedResult.Parts[j].Size {
-						t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size)
-					}
-					//  Asserting the ETag in the PartInfo.
-					if actualMetaData.ETag != expectedResult.Parts[j].ETag {
-						t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
-					}
-				}
-
 			}
 		}
 	}

cmd/object-api-utils.go

@@ -950,7 +950,7 @@ func compressSelfTest() {
 
 // getDiskInfos returns the disk information for the provided disks.
 // If a disk is nil or an error is returned the result will be nil as well.
-func getDiskInfos(ctx context.Context, disks []StorageAPI) []*DiskInfo {
+func getDiskInfos(ctx context.Context, disks ...StorageAPI) []*DiskInfo {
 	res := make([]*DiskInfo, len(disks))
 	for i, disk := range disks {
 		if disk == nil {

cmd/object-handlers.go

@@ -1424,6 +1424,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		srcInfo.metadataOnly = false
 	}
 
+	if srcInfo.metadataOnly {
+		gr.Close() // We are not interested in the reader stream at this point close it.
+	}
+
 	// Check if x-amz-metadata-directive or x-amz-tagging-directive was not set to REPLACE and source,
 	// destination are same objects. Apply this restriction also when
 	// metadataOnly is true indicating that we are not overwriting the object.
@@ -3449,17 +3453,15 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			writeErrorResponse(ctx, w, toAPIError(ctx, errors.New("force-delete is forbidden in a locked-enabled bucket")), r.URL)
 			return
 		}
-		if vID != "" {
-			apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
-				ObjectV: ObjectV{
-					ObjectName: object,
-					VersionID:  vID,
-				},
-			}, goi, gerr)
-			if apiErr != ErrNone && apiErr != ErrNoSuchKey {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
-				return
-			}
+		apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: object,
+				VersionID:  vID,
+			},
+		}, goi, gerr)
+		if apiErr != ErrNone && apiErr != ErrNoSuchKey {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
+			return
 		}
 	}
 

cmd/object-handlers_test.go

@@ -1954,7 +1954,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to look for copied object part: <ERROR> %s", i+1, instanceType, err)
 			}
-			if instanceType != FSTestStr && len(results.Parts) != 1 {
+			if len(results.Parts) != 1 {
 				t.Fatalf("Test %d: %s: Expected only one entry returned %d entries", i+1, instanceType, len(results.Parts))
 			}
 		}

cmd/prepare-storage.go

@@ -150,13 +150,13 @@ func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpo
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {
-			closeStorageDisks(storageDisks)
+			closeStorageDisks(storageDisks...)
 		}
 	}(storageDisks)
 
 	for i, err := range errs {
-		if err != nil {
-			if err == errDiskNotFound && verboseLogging {
+		if err != nil && !errors.Is(err, errXLBackend) {
+			if errors.Is(err, errDiskNotFound) && verboseLogging {
 				logger.Error("Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
 			} else {
 				logger.Error("Unable to use the drive %s: %v", endpoints[i], err)
@@ -173,7 +173,7 @@ func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpo
 	// Check if we have
 	for i, sErr := range sErrs {
 		// print the error, nonetheless, which is perhaps unhandled
-		if sErr != errUnformattedDisk && sErr != errDiskNotFound && verboseLogging {
+		if !errors.Is(sErr, errUnformattedDisk) && !errors.Is(sErr, errDiskNotFound) && verboseLogging {
 			if sErr != nil {
 				logger.Error("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)
 			}

cmd/server-main.go

@@ -73,6 +73,20 @@ var ServerFlags = []cli.Flag{
 		EnvVar: "MINIO_SHUTDOWN_TIMEOUT",
 		Hidden: true,
 	},
+	cli.DurationFlag{
+		Name:   "idle-timeout",
+		Value:  xhttp.DefaultIdleTimeout,
+		Usage:  "idle timeout is the maximum amount of time to wait for the next request when keep-alives are enabled",
+		EnvVar: "MINIO_IDLE_TIMEOUT",
+		Hidden: true,
+	},
+	cli.DurationFlag{
+		Name:   "read-header-timeout",
+		Value:  xhttp.DefaultReadHeaderTimeout,
+		Usage:  "read header timeout is the amount of time allowed to read request headers",
+		EnvVar: "MINIO_READ_HEADER_TIMEOUT",
+		Hidden: true,
+	},
 }
 
 var serverCmd = cli.Command{
@@ -208,6 +222,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	if globalIsDistErasure {
 		globalIsErasure = true
 	}
+	globalIsErasureSD = (setupType == ErasureSDSetupType)
 }
 
 func serverHandleEnvVars() {
@@ -218,13 +233,11 @@ func serverHandleEnvVars() {
 var globalHealStateLK sync.RWMutex
 
 func initAllSubsystems() {
-	if globalIsErasure {
-		globalHealStateLK.Lock()
-		// New global heal state
-		globalAllHealState = newHealState(true)
-		globalBackgroundHealState = newHealState(false)
-		globalHealStateLK.Unlock()
-	}
+	globalHealStateLK.Lock()
+	// New global heal state
+	globalAllHealState = newHealState(true)
+	globalBackgroundHealState = newHealState(false)
+	globalHealStateLK.Unlock()
 
 	// Create new notification system and initialize notification peer targets
 	globalNotificationSys = NewNotificationSys(globalEndpoints)
@@ -264,8 +277,6 @@ func initAllSubsystems() {
 	// Create new bucket versioning subsystem
 	if globalBucketVersioningSys == nil {
 		globalBucketVersioningSys = NewBucketVersioningSys()
-	} else {
-		globalBucketVersioningSys.Reset()
 	}
 
 	// Create new bucket replication subsytem
@@ -486,6 +497,8 @@ func serverMain(ctx *cli.Context) {
 		UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 		UseTLSConfig(newTLSConfig(getCert)).
 		UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
+		UseIdleTimeout(ctx.Duration("idle-timeout")).
+		UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
 		UseBaseContext(GlobalContext).
 		UseCustomLogger(log.New(ioutil.Discard, "", 0)) // Turn-off random logging by Go stdlib
 
@@ -511,11 +524,8 @@ func serverMain(ctx *cli.Context) {
 	xhttp.SetMinIOVersion(Version)
 
 	// Enable background operations for erasure coding
-	if globalIsErasure {
-		initAutoHeal(GlobalContext, newObject)
-		initHealMRF(GlobalContext, newObject)
-	}
-
+	initAutoHeal(GlobalContext, newObject)
+	initHealMRF(GlobalContext, newObject)
 	initBackgroundExpiry(GlobalContext, newObject)
 
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
@@ -563,21 +573,19 @@ func serverMain(ctx *cli.Context) {
 	// Background all other operations such as initializing bucket metadata etc.
 	go func() {
 		// Initialize transition tier configuration manager
-		if globalIsErasure {
-			initBackgroundReplication(GlobalContext, newObject)
-			initBackgroundTransition(GlobalContext, newObject)
+		initBackgroundReplication(GlobalContext, newObject)
+		initBackgroundTransition(GlobalContext, newObject)
 
-			go func() {
-				if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
-					logger.LogIf(GlobalContext, err)
-				}
+		go func() {
+			if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
+				logger.LogIf(GlobalContext, err)
+			}
 
-				globalTierJournal, err = initTierDeletionJournal(GlobalContext)
-				if err != nil {
-					logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
-				}
-			}()
-		}
+			globalTierJournal, err = initTierDeletionJournal(GlobalContext)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
+			}
+		}()
 
 		// Initialize site replication manager.
 		globalSiteReplicationSys.Init(GlobalContext, newObject)
@@ -648,7 +656,13 @@ func newObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools
 	// For FS only, directly use the disk.
 	if endpointServerPools.NEndpoints() == 1 {
 		// Initialize new FS object layer.
-		return NewFSObjectLayer(endpointServerPools[0].Endpoints[0].Path)
+		newObject, err = NewFSObjectLayer(endpointServerPools[0].Endpoints[0].Path)
+		if err == nil {
+			return newObject, nil
+		}
+		if err != nil && err != errFreshDisk {
+			return newObject, err
+		}
 	}
 
 	return newErasureServerPools(ctx, endpointServerPools)

cmd/server-main_test.go

@@ -27,7 +27,7 @@ import (
 func TestNewObjectLayer(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	// Tests for FS object layer.
+	// Tests for ErasureSD object layer.
 	nDisks := 1
 	disks, err := getRandomDisks(nDisks)
 	if err != nil {
@@ -39,7 +39,7 @@ func TestNewObjectLayer(t *testing.T) {
 	if err != nil {
 		t.Fatal("Unexpected object layer initialization error", err)
 	}
-	_, ok := obj.(*FSObjects)
+	_, ok := obj.(*erasureSingle)
 	if !ok {
 		t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj))
 	}

cmd/server_test.go

@@ -39,7 +39,7 @@ import (
 	"github.com/minio/pkg/bucket/policy"
 )
 
-// API suite container common to both FS and Erasure.
+// API suite container common to both ErasureSD and Erasure.
 type TestSuiteCommon struct {
 	serverType string
 	testServer TestServer
@@ -122,12 +122,12 @@ func runAllTests(suite *TestSuiteCommon, c *check) {
 
 func TestServerSuite(t *testing.T) {
 	testCases := []*TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend with signature v2.
-		{serverType: "FS", signer: signerV2},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on ErasureSD backend with signature v4.
+		{serverType: "ErasureSD", signer: signerV4},
+		// Init and run test on ErasureSD backend with signature v2.
+		{serverType: "ErasureSD", signer: signerV2},
+		// Init and run test on ErasureSD backend, with tls enabled.
+		{serverType: "ErasureSD", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
 		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.

cmd/setup-type.go

@@ -27,6 +27,9 @@ const (
 	// FSSetupType - FS setup type enum.
 	FSSetupType
 
+	// ErasureSDSetupType - Erasure single drive setup enum.
+	ErasureSDSetupType
+
 	// ErasureSetupType - Erasure setup type enum.
 	ErasureSetupType
 
@@ -41,6 +44,8 @@ func (setupType SetupType) String() string {
 	switch setupType {
 	case FSSetupType:
 		return globalMinioModeFS
+	case ErasureSDSetupType:
+		return globalMinioModeErasureSD
 	case ErasureSetupType:
 		return globalMinioModeErasure
 	case DistErasureSetupType:

cmd/signature-v4-utils.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"crypto/hmac"
-	"crypto/sha256"
 	"encoding/hex"
 	"io"
 	"io/ioutil"
@@ -29,6 +28,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 )

cmd/signature-v4.go

@@ -27,7 +27,6 @@ package cmd
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"crypto/subtle"
 	"encoding/hex"
 	"net/http"
@@ -40,6 +39,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 )
 

cmd/site-replication.go

@@ -876,14 +876,28 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 				ExistingObjectReplicate: "enable",
 			}
 		)
+		ruleARN := targetARN
 		for _, r := range replicationConfig.Rules {
 			if r.ID == ruleID {
 				hasRule = true
+				ruleARN = r.Destination.Bucket
 			}
 		}
 		switch {
 		case hasRule:
-			err = replicationConfig.EditRule(opts)
+			if ruleARN != opts.DestBucket {
+				// remove stale replication rule and replace rule with correct target ARN
+				if len(replicationConfig.Rules) > 1 {
+					err = replicationConfig.RemoveRule(opts)
+				} else {
+					replicationConfig = replication.Config{}
+				}
+				if err == nil {
+					err = replicationConfig.AddRule(opts)
+				}
+			} else {
+				err = replicationConfig.EditRule(opts)
+			}
 		default:
 			err = replicationConfig.AddRule(opts)
 		}
@@ -902,7 +916,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		if err != nil {
 			return err
 		}
-		sameTarget, apiErr := validateReplicationDestination(ctx, bucket, newReplicationConfig)
+		sameTarget, apiErr := validateReplicationDestination(ctx, bucket, newReplicationConfig, true)
 		if apiErr != noError {
 			return fmt.Errorf("bucket replication config validation error: %#v", apiErr)
 		}
@@ -2970,8 +2984,9 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 				CreatedAt: createdAt,
 				Location:  globalSite.Region,
 			}
+
 			// Get bucket policy if present.
-			policy, err := globalPolicySys.Get(bucket)
+			policy, updatedAt, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
 			found := true
 			if _, ok := err.(BucketPolicyNotFound); ok {
 				found = false
@@ -2984,6 +2999,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 					return info, wrapSRErr(err)
 				}
 				bms.Policy = policyJSON
+				bms.PolicyUpdatedAt = updatedAt
 			}
 
 			// Get bucket tags if present.
@@ -3004,6 +3020,23 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 				bms.TagConfigUpdatedAt = updatedAt
 			}
 
+			versioningCfg, updatedAt, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+			found = true
+			if versioningCfg != nil && versioningCfg.Status == "" {
+				found = false
+			} else if err != nil {
+				return info, errSRBackendIssue(err)
+			}
+			if found {
+				versionCfgData, err := xml.Marshal(versioningCfg)
+				if err != nil {
+					return info, wrapSRErr(err)
+				}
+				versioningCfgStr := base64.StdEncoding.EncodeToString(versionCfgData)
+				bms.Versioning = &versioningCfgStr
+				bms.VersioningConfigUpdatedAt = updatedAt
+			}
+
 			// Get object-lock config if present.
 			objLockCfg, updatedAt, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 			found = true
@@ -3111,13 +3144,17 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			}
 		} else {
 			globalIAMSys.store.rlock()
-			if err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap); err != nil {
-				return info, errSRBackendIssue(err)
-			}
-			if err = globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap); err != nil {
-				return info, errSRBackendIssue(err)
-			}
+			stsErr := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
 			globalIAMSys.store.runlock()
+			if stsErr != nil {
+				return info, errSRBackendIssue(stsErr)
+			}
+			globalIAMSys.store.rlock()
+			usrErr := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
+			globalIAMSys.store.runlock()
+			if usrErr != nil {
+				return info, errSRBackendIssue(usrErr)
+			}
 		}
 		info.UserPolicies = make(map[string]madmin.SRPolicyMapping, len(userPolicyMap))
 		for user, mp := range userPolicyMap {
@@ -3172,13 +3209,17 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			}
 		} else {
 			globalIAMSys.store.rlock()
-			if err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap); err != nil {
-				return info, errSRBackendIssue(err)
-			}
-			if err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap); err != nil {
-				return info, errSRBackendIssue(err)
-			}
+			stsErr := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
 			globalIAMSys.store.runlock()
+			if stsErr != nil {
+				return info, errSRBackendIssue(stsErr)
+			}
+			globalIAMSys.store.rlock()
+			userErr := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
+			globalIAMSys.store.runlock()
+			if userErr != nil {
+				return info, errSRBackendIssue(userErr)
+			}
 		}
 
 		info.GroupPolicies = make(map[string]madmin.SRPolicyMapping, len(c.state.Peers))
@@ -3560,7 +3601,6 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 		latestID, latestPeerName string
 		lastUpdate               time.Time
 		latestQuotaConfig        *string
-		latestCfgJSON            []byte
 		latestQuotaConfigBytes   []byte
 	)
 
@@ -3614,7 +3654,7 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 		if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
 			Type:   madmin.SRBucketMetaTypeQuotaConfig,
 			Bucket: bucket,
-			Quota:  latestCfgJSON,
+			Quota:  latestQuotaConfigBytes,
 		}); err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing quota config metadata for peer %s from peer %s : %s",
 				peerName, latestPeerName, err.Error())))
@@ -3949,6 +3989,16 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
 			break
 		}
 	}
+	rcfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		replMismatch = true
+	}
+
+	// validate remote targets on current cluster for this bucket
+	_, apiErr := validateReplicationDestination(ctx, bucket, rcfg, false)
+	if apiErr != noError {
+		replMismatch = true
+	}
 	if replMismatch {
 		err := c.PeerBucketConfigureReplHandler(ctx, bucket)
 		if err != nil {

cmd/storage-errors.go

@@ -116,6 +116,12 @@ var errDoneForNow = errors.New("done for now")
 // to proceed to next entry.
 var errSkipFile = errors.New("skip this file")
 
+// Returned by FS drive mode when a fresh disk is specified.
+var errFreshDisk = errors.New("FS backend requires existing disk")
+
+// errXLBackend XL drive mode requires fresh deployment.
+var errXLBackend = errors.New("XL backend requires fresh disk")
+
 // StorageErr represents error generated by xlStorage call.
 type StorageErr string
 

cmd/storage-rest-server.go

@@ -1140,6 +1140,10 @@ func checkDiskFatalErrs(errs []error) error {
 		return errFaultyDisk
 	}
 
+	if countErrs(errs, errXLBackend) == len(errs) {
+		return errXLBackend
+	}
+
 	return nil
 }
 
@@ -1152,6 +1156,8 @@ func checkDiskFatalErrs(errs []error) error {
 // Do not like it :-(
 func logFatalErrs(err error, endpoint Endpoint, exit bool) {
 	switch {
+	case errors.Is(err, errXLBackend):
+		logger.Fatal(config.ErrInvalidXLValue(err), "Unable to initialize backend")
 	case errors.Is(err, errUnsupportedDisk):
 		var hint string
 		if endpoint.URL != nil {

cmd/streaming-signature-v4.go

@@ -22,7 +22,6 @@ package cmd
 import (
 	"bufio"
 	"bytes"
-	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"hash"
@@ -32,6 +31,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 )
 

cmd/sts-datatypes.go

@@ -203,3 +203,16 @@ type AssumeRoleWithCertificateResponse struct {
 		RequestID string `xml:"RequestId,omitempty"`
 	} `xml:"ResponseMetadata,omitempty"`
 }
+
+// AssumeRoleWithCustomTokenResponse contains the result of a successful
+// AssumeRoleWithCustomToken request.
+type AssumeRoleWithCustomTokenResponse struct {
+	XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithCustomTokenResponse" json:"-"`
+	Result  struct {
+		Credentials auth.Credentials `xml:"Credentials,omitempty"`
+		AssumedUser string           `xml:"AssumedUser,omitempty"`
+	} `xml:"AssumeRoleWithCustomTokenResult"`
+	Metadata struct {
+		RequestID string `xml:"RequestId,omitempty"`
+	} `xml:"ResponseMetadata,omitempty"`
+}

cmd/sts-errors.go

@@ -96,6 +96,7 @@ const (
 	ErrSTSInsecureConnection
 	ErrSTSInvalidClientCertificate
 	ErrSTSNotInitialized
+	ErrSTSUpstreamError
 	ErrSTSInternalError
 )
 
@@ -162,6 +163,11 @@ var stsErrCodes = stsErrorCodeMap{
 		Description:    "STS API not initialized, please try again.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrSTSUpstreamError: {
+		Code:           "InternalError",
+		Description:    "An upstream service required for this operation failed - please try again or contact an administrator.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
 	ErrSTSInternalError: {
 		Code:           "InternalError",
 		Description:    "We encountered an internal error generating credentials, please try again.",

cmd/sts-handlers.go

@@ -20,12 +20,12 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"crypto/x509"
 	"encoding/base64"
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -33,6 +33,7 @@ import (
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -54,11 +55,12 @@ const (
 	stsLDAPPassword           = "LDAPPassword"
 
 	// STS API action constants
-	clientGrants      = "AssumeRoleWithClientGrants"
-	webIdentity       = "AssumeRoleWithWebIdentity"
-	ldapIdentity      = "AssumeRoleWithLDAPIdentity"
-	clientCertificate = "AssumeRoleWithCertificate"
-	assumeRole        = "AssumeRole"
+	clientGrants        = "AssumeRoleWithClientGrants"
+	webIdentity         = "AssumeRoleWithWebIdentity"
+	ldapIdentity        = "AssumeRoleWithLDAPIdentity"
+	clientCertificate   = "AssumeRoleWithCertificate"
+	customTokenIdentity = "AssumeRoleWithCustomToken"
+	assumeRole          = "AssumeRole"
 
 	stsRequestBodyLimit = 10 * (1 << 20) // 10 MiB
 
@@ -128,6 +130,11 @@ func registerSTSRouter(router *mux.Router) {
 	stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithCertificate)).
 		Queries(stsAction, clientCertificate).
 		Queries(stsVersion, stsAPIVersion)
+
+	// AssumeRoleWithCustomToken
+	stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithCustomToken)).
+		Queries(stsAction, customTokenIdentity).
+		Queries(stsVersion, stsAPIVersion)
 }
 
 func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Credentials, isErrCodeSTS bool, stsErr STSErrorCode) {
@@ -382,7 +389,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 			policyName = globalIAMSys.CurrentPolicies(policies)
 		}
 
-		if globalAuthZPlugin == nil {
+		if newGlobalAuthZPluginFn() == nil {
 			if !ok {
 				writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
 					fmt.Errorf("%s claim missing from the JWT token, credentials will not be generated", iamPolicyClaimNameOpenID()))
@@ -591,7 +598,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 
 	// Check if this user or their groups have a policy applied.
 	ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, false, groupDistNames...)
-	if len(ldapPolicies) == 0 && globalAuthZPlugin == nil {
+	if len(ldapPolicies) == 0 && newGlobalAuthZPluginFn() == nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
 			fmt.Errorf("expecting a policy to be set for user `%s` or one of their groups: `%s` - rejecting this request",
 				ldapUserDN, strings.Join(groupDistNames, "`,`")))
@@ -815,3 +822,125 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	response.Metadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
+
+// AssumeRoleWithCustomToken implements user authentication with custom tokens.
+// These tokens are opaque to MinIO and are verified by a configured (external)
+// Identity Management Plugin.
+//
+// API endpoint: https://minio:9000?Action=AssumeRoleWithCustomToken&Token=xxx
+func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRoleWithCustomToken")
+
+	if globalAuthNPlugin == nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCustomToken' is disabled"))
+		return
+	}
+
+	action := r.Form.Get(stsAction)
+	if action != customTokenIdentity {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
+		return
+	}
+
+	defer logger.AuditLog(ctx, w, r, nil)
+
+	token := r.Form.Get(stsToken)
+	if token == "" {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid empty `Token` parameter provided"))
+		return
+	}
+
+	durationParam := r.Form.Get(stsDurationSeconds)
+	var requestedDuration int
+	if durationParam != "" {
+		var err error
+		requestedDuration, err = strconv.Atoi(durationParam)
+		if err != nil {
+			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid requested duration: %s", durationParam))
+			return
+		}
+	}
+
+	roleArnStr := r.Form.Get(stsRoleArn)
+	roleArn, _, err := globalIAMSys.GetRolePolicy(roleArnStr)
+	if err != nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+			fmt.Errorf("Error processing parameter %s: %v", stsRoleArn, err))
+		return
+	}
+
+	res, err := globalAuthNPlugin.Authenticate(roleArn, token)
+	if err != nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		return
+	}
+
+	// If authentication failed, return the error message to the user.
+	if res.Failure != nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSUpstreamError, errors.New(res.Failure.Reason))
+		return
+	}
+
+	// It is required that parent user be set.
+	if res.Success.User == "" {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSUpstreamError, errors.New("A valid user was not returned by the authenticator."))
+		return
+	}
+
+	// Expiry is set as minimum of requested value and value allowed by auth
+	// plugin.
+	expiry := res.Success.MaxValiditySeconds
+	if durationParam != "" && requestedDuration < expiry {
+		expiry = requestedDuration
+	}
+
+	parentUser := "custom:" + res.Success.User
+
+	// metadata map
+	m := map[string]interface{}{
+		expClaim:     UTCNow().Add(time.Duration(expiry) * time.Second).Unix(),
+		parentClaim:  parentUser,
+		subClaim:     parentUser,
+		roleArnClaim: roleArn.String(),
+	}
+	// Add all other claims from the plugin **without** replacing any
+	// existing claims.
+	for k, v := range res.Success.Claims {
+		if _, ok := m[k]; !ok {
+			m[k] = v
+		}
+	}
+
+	tmpCredentials, err := auth.GetNewCredentialsWithMetadata(m, globalActiveCred.SecretKey)
+	if err != nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		return
+	}
+
+	tmpCredentials.ParentUser = parentUser
+	err = globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, "")
+	if err != nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		return
+	}
+
+	// Call hook for site replication.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemSTSAcc,
+		STSCredential: &madmin.SRSTSCredential{
+			AccessKey:    tmpCredentials.AccessKey,
+			SecretKey:    tmpCredentials.SecretKey,
+			SessionToken: tmpCredentials.SessionToken,
+			ParentUser:   tmpCredentials.ParentUser,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	response := new(AssumeRoleWithCustomTokenResponse)
+	response.Result.Credentials = tmpCredentials
+	response.Result.AssumedUser = parentUser
+	response.Metadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
+	writeSuccessResponseXML(w, encodeResponse(response))
+}

cmd/sts-handlers_test.go

@@ -42,10 +42,10 @@ func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
 
 func TestIAMInternalIDPSTSServerSuite(t *testing.T) {
 	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on ErasureSD backend with signature v4.
+		{serverType: "ErasureSD", signer: signerV4},
+		// Init and run test on ErasureSD backend, with tls enabled.
+		{serverType: "ErasureSD", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
 		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.

cmd/stserrorcode_string.go

@@ -19,12 +19,13 @@ func _() {
 	_ = x[ErrSTSInsecureConnection-8]
 	_ = x[ErrSTSInvalidClientCertificate-9]
 	_ = x[ErrSTSNotInitialized-10]
-	_ = x[ErrSTSInternalError-11]
+	_ = x[ErrSTSUpstreamError-11]
+	_ = x[ErrSTSInternalError-12]
 }
 
-const _STSErrorCode_name = "STSNoneSTSAccessDeniedSTSMissingParameterSTSInvalidParameterValueSTSWebIdentityExpiredTokenSTSClientGrantsExpiredTokenSTSInvalidClientGrantsTokenSTSMalformedPolicyDocumentSTSInsecureConnectionSTSInvalidClientCertificateSTSNotInitializedSTSInternalError"
+const _STSErrorCode_name = "STSNoneSTSAccessDeniedSTSMissingParameterSTSInvalidParameterValueSTSWebIdentityExpiredTokenSTSClientGrantsExpiredTokenSTSInvalidClientGrantsTokenSTSMalformedPolicyDocumentSTSInsecureConnectionSTSInvalidClientCertificateSTSNotInitializedSTSUpstreamErrorSTSInternalError"
 
-var _STSErrorCode_index = [...]uint8{0, 7, 22, 41, 65, 91, 118, 145, 171, 192, 219, 236, 252}
+var _STSErrorCode_index = [...]uint16{0, 7, 22, 41, 65, 91, 118, 145, 171, 192, 219, 236, 252, 268}
 
 func (i STSErrorCode) String() string {
 	if i < 0 || i >= STSErrorCode(len(_STSErrorCode_index)-1) {

cmd/test-utils_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bufio"
 	"bytes"
 	"context"
@@ -46,6 +47,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strconv"
@@ -76,6 +78,8 @@ func TestMain(m *testing.M) {
 	// set to 'true' when testing is invoked
 	globalIsTesting = true
 
+	globalIsCICD = globalIsTesting
+
 	globalActiveCred = auth.Credentials{
 		AccessKey: auth.DefaultAccessKey,
 		SecretKey: auth.DefaultSecretKey,
@@ -189,10 +193,14 @@ func prepareFS() (ObjectLayer, string, error) {
 	if err != nil {
 		return nil, "", err
 	}
-	obj, err := NewFSObjectLayer(fsDirs[0])
+	obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(fsDirs...))
 	if err != nil {
 		return nil, "", err
 	}
+
+	initAllSubsystems()
+
+	globalIAMSys.Init(context.Background(), obj, globalEtcdClient, 2*time.Second)
 	return obj, fsDirs[0], nil
 }
 
@@ -219,8 +227,7 @@ func prepareErasure16(ctx context.Context) (ObjectLayer, []string, error) {
 
 // Initialize FS objects.
 func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) {
-	var err error
-	obj, err = NewFSObjectLayer(disk)
+	obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(disk))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -240,8 +247,8 @@ type TestErrHandler interface {
 }
 
 const (
-	// FSTestStr is the string which is used as notation for Single node ObjectLayer in the unit tests.
-	FSTestStr string = "FS"
+	// ErasureSDStr is the string which is used as notation for Single node ObjectLayer in the unit tests.
+	ErasureSDStr string = "ErasureSD"
 
 	// ErasureTestStr is the string which is used as notation for Erasure ObjectLayer in the unit tests.
 	ErasureTestStr string = "Erasure"
@@ -1467,20 +1474,9 @@ func getRandomDisks(N int) ([]string, error) {
 
 // Initialize object layer with the supplied disks, objectLayer is nil upon any error.
 func newTestObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools) (newObject ObjectLayer, err error) {
-	// For FS only, directly use the disk.
-	if endpointServerPools.NEndpoints() == 1 {
-		// Initialize new FS object layer.
-		return NewFSObjectLayer(endpointServerPools[0].Endpoints[0].Path)
-	}
-
-	z, err := newErasureServerPools(ctx, endpointServerPools)
-	if err != nil {
-		return nil, err
-	}
-
 	initAllSubsystems()
 
-	return z, nil
+	return newErasureServerPools(ctx, endpointServerPools)
 }
 
 // initObjectLayer - Instantiates object layer and returns it.
@@ -1748,7 +1744,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	credentials := globalActiveCred
 
 	// Executing the object layer tests for single node setup.
-	objAPITest(objLayer, FSTestStr, bucketFS, fsAPIRouter, credentials, t)
+	objAPITest(objLayer, ErasureSDStr, bucketFS, fsAPIRouter, credentials, t)
 
 	objLayer, erasureDisks, err := prepareErasure16(ctx)
 	if err != nil {
@@ -1814,7 +1810,7 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
 		globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
 
 		// Executing the object layer tests for single node setup.
-		objTest(objLayer, FSTestStr, t)
+		objTest(objLayer, ErasureSDStr, t)
 
 		// Call clean up functions
 		cancel()
@@ -2323,3 +2319,35 @@ func uploadTestObject(t *testing.T, apiRouter http.Handler, creds auth.Credentia
 		checkRespErr(rec, http.StatusOK)
 	}
 }
+
+// unzip a file into a specific target dir - used to unzip sample data in cmd/testdata/
+func unzipArchive(zipFilePath, targetDir string) error {
+	zipReader, err := zip.OpenReader(zipFilePath)
+	if err != nil {
+		return err
+	}
+	for _, file := range zipReader.Reader.File {
+		zippedFile, err := file.Open()
+		if err != nil {
+			return err
+		}
+		err = func() (err error) {
+			defer zippedFile.Close()
+			extractedFilePath := filepath.Join(targetDir, file.Name)
+			if file.FileInfo().IsDir() {
+				return os.MkdirAll(extractedFilePath, file.Mode())
+			}
+			outputFile, err := os.OpenFile(extractedFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+			if err != nil {
+				return err
+			}
+			defer outputFile.Close()
+			_, err = io.Copy(outputFile, zippedFile)
+			return err
+		}()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/tier-handlers.go

@@ -74,7 +74,7 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -132,7 +132,7 @@ func (api adminAPIHandlers) ListTierHandler(w http.ResponseWriter, r *http.Reque
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -158,7 +158,7 @@ func (api adminAPIHandlers) EditTierHandler(w http.ResponseWriter, r *http.Reque
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -210,7 +210,7 @@ func (api adminAPIHandlers) RemoveTierHandler(w http.ResponseWriter, r *http.Req
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -247,7 +247,7 @@ func (api adminAPIHandlers) VerifyTierHandler(w http.ResponseWriter, r *http.Req
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -273,7 +273,7 @@ func (api adminAPIHandlers) TierStatsHandler(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}

cmd/utils.go

@@ -906,6 +906,8 @@ func getMinioMode() string {
 		mode = globalMinioModeErasure
 	} else if globalIsGateway {
 		mode = globalMinioModeGatewayPrefix + globalGatewayName
+	} else if globalIsErasureSD {
+		mode = globalMinioModeErasureSD
 	}
 	return mode
 }

cmd/xl-storage-format-utils.go

@@ -50,7 +50,9 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
 	var versions []FileInfo
 	var err error
 
-	if buf, _ := isIndexedMetaV2(xlMetaBuf); buf != nil {
+	if buf, _, e := isIndexedMetaV2(xlMetaBuf); e != nil {
+		return FileInfoVersions{}, e
+	} else if buf != nil {
 		versions, err = buf.ListVersions(volume, path)
 	} else {
 		var xlMeta xlMetaV2
@@ -87,7 +89,9 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
 	var fi FileInfo
 	var err error
 	var inData xlMetaInlineData
-	if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
+	if buf, data, e := isIndexedMetaV2(xlMetaBuf); e != nil {
+		return FileInfo{}, e
+	} else if buf != nil {
 		inData = data
 		fi, err = buf.ToFileInfo(volume, path, versionID)
 		if len(buf) != 0 && errors.Is(err, errFileNotFound) {
@@ -141,6 +145,9 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
 // Will return -1 for unknown values.
 func getXLDiskLoc(diskID string) (poolIdx, setIdx, diskIdx int) {
 	if api := newObjectLayerFn(); api != nil {
+		if globalIsErasureSD {
+			return 0, 0, 0
+		}
 		if ep, ok := api.(*erasureServerPools); ok {
 			if pool, set, disk, err := ep.getPoolAndSet(diskID); err == nil {
 				return pool, set, disk

cmd/xl-storage-format-v1.go

@@ -53,7 +53,7 @@ func isXLMetaFormatValid(version, format string) bool {
 // Verifies if the backend format metadata is sane by validating
 // the ErasureInfo, i.e. data and parity blocks.
 func isXLMetaErasureInfoValid(data, parity int) bool {
-	return ((data >= parity) && (data != 0) && (parity != 0))
+	return ((data >= parity) && (data > 0) && (parity >= 0))
 }
 
 //go:generate msgp -file=$GOFILE -unexported

cmd/xl-storage-format-v2.go

@@ -793,34 +793,32 @@ func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte)
 }
 
 // isIndexedMetaV2 returns non-nil result if metadata is indexed.
-// If data doesn't validate nil is also returned.
-func isIndexedMetaV2(buf []byte) (meta xlMetaBuf, data xlMetaInlineData) {
+// Returns 3x nil if not XLV2 or not indexed.
+// If indexed and unable to parse an error will be returned.
+func isIndexedMetaV2(buf []byte) (meta xlMetaBuf, data xlMetaInlineData, err error) {
 	buf, major, minor, err := checkXL2V1(buf)
-	if err != nil {
-		return nil, nil
-	}
-	if major != 1 || minor < 3 {
-		return nil, nil
+	if err != nil || major != 1 || minor < 3 {
+		return nil, nil, nil
 	}
 	meta, buf, err = msgp.ReadBytesZC(buf)
 	if err != nil {
-		return nil, nil
+		return nil, nil, err
 	}
 	if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil {
 		// Read metadata CRC
 		buf = nbuf
 		if got := uint32(xxhash.Sum64(meta)); got != crc {
-			return nil, nil
+			return nil, nil, fmt.Errorf("xlMetaV2.Load version(%d), CRC mismatch, want 0x%x, got 0x%x", minor, crc, got)
 		}
 	} else {
-		return nil, nil
+		return nil, nil, err
 	}
 	data = buf
 	if data.validate() != nil {
 		data.repair()
 	}
 
-	return meta, data
+	return meta, data, nil
 }
 
 type xlMetaV2ShallowVersion struct {
@@ -865,7 +863,9 @@ func (x *xlMetaV2) LoadOrConvert(buf []byte) error {
 // Load all versions of the stored data.
 // Note that references to the incoming buffer will be kept.
 func (x *xlMetaV2) Load(buf []byte) error {
-	if meta, data := isIndexedMetaV2(buf); meta != nil {
+	if meta, data, err := isIndexedMetaV2(buf); err != nil {
+		return err
+	} else if meta != nil {
 		return x.loadIndexed(meta, data)
 	}
 	// Convert older format.

cmd/xl-storage-format_test.go

@@ -62,13 +62,15 @@ func TestIsXLMetaErasureInfoValid(t *testing.T) {
 		{1, 5, 6, false},
 		{2, 5, 5, true},
 		{3, 0, 5, false},
-		{4, 5, 0, false},
-		{5, 5, 0, false},
-		{6, 5, 4, true},
+		{3, -1, 5, false},
+		{4, 5, -1, false},
+		{5, 5, 0, true},
+		{6, 5, 0, true},
+		{7, 5, 4, true},
 	}
 	for _, tt := range tests {
 		if got := isXLMetaErasureInfoValid(tt.data, tt.parity); got != tt.want {
-			t.Errorf("Test %d: Expected %v but received %v", tt.name, got, tt.want)
+			t.Errorf("Test %d: Expected %v but received %v -> %#v", tt.name, got, tt.want, tt)
 		}
 	}
 }
@@ -512,7 +514,7 @@ func BenchmarkXlMetaV2Shallow(b *testing.B) {
 				b.ResetTimer()
 				b.ReportAllocs()
 				for i := 0; i < b.N; i++ {
-					buf, _ := isIndexedMetaV2(enc)
+					buf, _, _ := isIndexedMetaV2(enc)
 					if buf == nil {
 						b.Fatal("buf == nil")
 					}
@@ -527,7 +529,7 @@ func BenchmarkXlMetaV2Shallow(b *testing.B) {
 				b.ResetTimer()
 				b.ReportAllocs()
 				for i := 0; i < b.N; i++ {
-					buf, _ := isIndexedMetaV2(enc)
+					buf, _, _ := isIndexedMetaV2(enc)
 					if buf == nil {
 						b.Fatal("buf == nil")
 					}

cmd/xl-storage.go

@@ -443,6 +443,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			}
 		}
 	}
+
+	vcfg, _ := globalBucketVersioningSys.Get(cache.Info.Name)
+
 	// return initialized object layer
 	objAPI := newObjectLayerFn()
 	// object layer not initialized, return.
@@ -494,9 +497,12 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			}
 			return sizeSummary{}, errSkipFile
 		}
+
+		versioned := vcfg != nil && vcfg.Versioned(item.objectPath())
+
 		for _, version := range fivs.Versions {
 			atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
-			oi := version.ToObjectInfo(item.bucket, item.objectPath())
+			oi := version.ToObjectInfo(item.bucket, item.objectPath(), versioned)
 			sz := item.applyActions(ctx, objAPI, oi, &sizeS)
 			if oi.VersionID != "" && sz == oi.Size {
 				sizeS.versions++
@@ -521,7 +527,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 
 		// apply tier sweep action on free versions
 		for _, freeVersion := range fivs.FreeVersions {
-			oi := freeVersion.ToObjectInfo(item.bucket, item.objectPath())
+			oi := freeVersion.ToObjectInfo(item.bucket, item.objectPath(), versioned)
 			item.applyTierObjSweep(ctx, objAPI, oi)
 		}
 		return sizeS, nil

docs/config/README.md

@@ -309,7 +309,7 @@ Example: The following settings will increase the heal operation speed by allowi
 
 Once set the healer settings are automatically applied without the need for server restarts.
 
-> NOTE: Healing is not supported for gateway and single drive mode.
+> NOTE: Healing is not supported for Gateway deployments.
 
 ## Environment only settings (not in config)
 

docs/debugging/hash-set/main.go

@@ -18,11 +18,16 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/binary"
 	"flag"
 	"fmt"
 	"hash/crc32"
+	"io/ioutil"
 	"log"
+	"os"
+	"strings"
 
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
@@ -64,22 +69,22 @@ func hashOrder(key string, cardinality int) []int {
 }
 
 var (
-	object, deploymentID string
-	setCount, shards     int
+	file, object, deploymentID, prefix string
+	setCount, shards                   int
+	verbose                            bool
 )
 
 func main() {
+	flag.StringVar(&file, "file", "", "Read all objects from file, newline separated")
+	flag.StringVar(&prefix, "prefix", "", "Add prefix to all objects")
 	flag.StringVar(&object, "object", "", "Select an object")
 	flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'")
 	flag.IntVar(&setCount, "set-count", 0, "Total set count")
 	flag.IntVar(&shards, "shards", 0, "Total shards count")
+	flag.BoolVar(&verbose, "v", false, "Display all objects")
 
 	flag.Parse()
 
-	if object == "" {
-		log.Fatalln("object name is mandatory")
-	}
-
 	if deploymentID == "" {
 		log.Fatalln("deployment ID is mandatory")
 	}
@@ -88,12 +93,39 @@ func main() {
 		log.Fatalln("set count cannot be zero")
 	}
 
-	if shards == 0 {
-		log.Fatalln("total shards cannot be zero")
-	}
-
 	id := uuid.MustParse(deploymentID)
 
-	fmt.Println("Erasure distribution for the object", hashOrder(object, shards))
-	fmt.Println("Erasure setNumber for the object", sipHashMod(object, setCount, id)+1)
+	if file != "" {
+		distrib := make([][]string, setCount)
+		b, err := ioutil.ReadFile(file)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		b = bytes.ReplaceAll(b, []byte("\r"), []byte{})
+		sc := bufio.NewScanner(bytes.NewBuffer(b))
+		for sc.Scan() {
+			object = strings.TrimSpace(sc.Text())
+			set := sipHashMod(prefix+object, setCount, id)
+			distrib[set] = append(distrib[set], prefix+object)
+		}
+		for set, files := range distrib {
+			fmt.Println("Set:", set+1, "Objects:", len(files))
+			if !verbose {
+				continue
+			}
+			for _, s := range files {
+				fmt.Printf("\t%s\n", s)
+			}
+		}
+		os.Exit(0)
+	}
+
+	if object == "" {
+		log.Fatalln("object name is mandatory")
+	}
+
+	if shards != 0 {
+		fmt.Println("Erasure distribution for the object", hashOrder(prefix+object, shards))
+	}
+	fmt.Println("Erasure setNumber for the object", sipHashMod(prefix+object, setCount, id)+1)
 }

docs/debugging/s3-check-md5/main.go

@@ -38,6 +38,7 @@ var (
 	bucket, prefix                 string
 	debug                          bool
 	versions                       bool
+	insecure                       bool
 )
 
 // getMD5Sum returns MD5 sum of given data.
@@ -55,6 +56,7 @@ func main() {
 	flag.StringVar(&prefix, "prefix", "", "Select a prefix")
 	flag.BoolVar(&debug, "debug", false, "Prints HTTP network calls to S3 endpoint")
 	flag.BoolVar(&versions, "versions", false, "Verify all versions")
+	flag.BoolVar(&insecure, "insecure", false, "Disable TLS verification")
 	flag.Parse()
 
 	if endpoint == "" {
@@ -78,12 +80,23 @@ func main() {
 		log.Fatalln(err)
 	}
 
+	secure := strings.EqualFold(u.Scheme, "https")
+	transport, err := minio.DefaultTransport(secure)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	if insecure {
+		// skip TLS verification
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	}
+
 	s3Client, err := minio.New(u.Host, &minio.Options{
-		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
-		Secure: strings.EqualFold(u.Scheme, "https"),
+		Creds:     credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure:    secure,
+		Transport: transport,
 	})
 	if err != nil {
-		log.Fatalln()
+		log.Fatalln(err)
 	}
 
 	if debug {
@@ -114,7 +127,7 @@ func main() {
 		// List all objects from a bucket-name with a matching prefix.
 		for object := range s3Client.ListObjects(context.Background(), bucket, opts) {
 			if object.Err != nil {
-				log.Fatalln("LIST error:", object.Err)
+				log.Println("LIST error:", object.Err)
 				continue
 			}
 			if object.IsDeleteMarker {
@@ -149,6 +162,7 @@ func main() {
 			}
 
 			var partsMD5Sum [][]byte
+			var failedMD5 bool
 			for p := 1; p <= parts; p++ {
 				opts := minio.GetObjectOptions{
 					VersionID:  object.VersionID,
@@ -157,16 +171,23 @@ func main() {
 				obj, err := s3Client.GetObject(context.Background(), bucket, object.Key, opts)
 				if err != nil {
 					log.Println("GET", bucket, object.Key, object.VersionID, "=>", err)
-					continue
+					failedMD5 = true
+					break
 				}
 				h := md5.New()
 				if _, err := io.Copy(h, obj); err != nil {
 					log.Println("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err)
-					continue
+					failedMD5 = true
+					break
 				}
 				partsMD5Sum = append(partsMD5Sum, h.Sum(nil))
 			}
 
+			if failedMD5 {
+				log.Println("CORRUPTED object:", bucket, object.Key, object.VersionID)
+				continue
+			}
+
 			corrupted := false
 			if !multipart {
 				md5sum := fmt.Sprintf("%x", partsMD5Sum[0])

docs/gateway/nas.md

@@ -1,4 +1,6 @@
-# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
+# MinIO NAS Gateway [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) 
+
+> NAS gateway is deprecated and will be removed in future, no more fresh deployments are supported.
 
 MinIO Gateway adds Amazon S3 compatibility to NAS storage. You may run multiple minio instances on the same shared NAS volume as a distributed object gateway.
 

docs/iam/identity-management-plugin.md

@@ -0,0 +1,76 @@
+# Identity Management Plugin Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+
+## Introduction
+
+To enable the integration of custom authentication methods, MinIO can be configured with an Identity Management Plugin webhook. When configured, this plugin enables the `AssumeRoleWithCustomToken` STS API extension. A user or application can now present a token to the `AssumeRoleWithCustomToken` API, and MinIO verifies this token by sending it to the Identity Management Plugin webhook. This plugin responds with some information and MinIO is able to generate temporary STS credentials to interact with object storage.
+
+The authentication flow is similar to that of OpenID, however the token is "opaque" to MinIO - it is simply sent to the plugin for verification. CAVEAT: There is no console UI integration for this method of authentication and it is intended primarily for machine authentication.
+
+It can be configured via MinIO's standard configuration API (i.e. using `mc admin config set/get`), or equivalently with environment variables. For brevity we show only environment variables here:
+
+```sh
+$ mc admin config set myminio identity_plugin --env
+KEY:
+identity_plugin  enable Identity Plugin via external hook
+
+ARGS:
+MINIO_IDENTITY_PLUGIN_URL*          (url)       plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/path/to/endpoint"
+MINIO_IDENTITY_PLUGIN_AUTH_TOKEN    (string)    authorization token for plugin hook endpoint
+MINIO_IDENTITY_PLUGIN_ROLE_POLICY*  (string)    policies to apply for plugin authorized users
+MINIO_IDENTITY_PLUGIN_ROLE_ID       (string)    unique ID to generate the ARN
+MINIO_IDENTITY_PLUGIN_COMMENT       (sentence)  optionally add a comment to this setting
+```
+
+If provided, the auth token parameter is sent as an authorization header.
+
+`MINIO_IDENTITY_PLUGIN_ROLE_POLICY` is a required parameter and can be list of comma separated policy names.
+
+On setting up the plugin, the MinIO server prints the Role ARN to its log. The Role ARN is generated by default based on the given plugin URL. To avoid this and use a configurable value set a unique role ID via `MINIO_IDENTITY_PLUGIN_ROLE_ID`.
+
+## REST API call to plugin
+
+To verify the custom token presented in the `AssumeRoleWithCustomToken` API, MinIO makes a POST request to the configured identity management plugin endpoint and expects a response with some details as shown below:
+
+### Request `POST` to plugin endpoint
+
+Query parameters:
+
+| Parameter Name | Value Type | Purpose                                                                 |
+|----------------|------------|-------------------------------------------------------------------------|
+| token          | string     | Token from the AssumeRoleWithCustomToken call for external verification |
+
+### Response
+
+If the token is valid and access is approved, the plugin must return a `200` (OK) HTTP status code.
+
+A `200 OK` Response should have `application/json` content-type and body with the following structure:
+
+```json
+{
+    "user": <string>,
+    "maxValiditySeconds": <integer>,
+    "claims": <key-value-pairs>
+}
+```
+
+| Parameter Name     | Value Type                              | Purpose                                                |
+|--------------------|-----------------------------------------|--------------------------------------------------------|
+| user               | string                                  | Identifier for owner of requested credentials          |
+| maxValiditySeconds | integer (>= 900 seconds and < 365 days) | Maximum allowed expiry duration for the credentials    |
+| claims             | key-value pairs                         | Claims to be associated with the requested credentials |
+
+The keys "exp", "parent" and "sub" in the `claims` object are reserved and if present are ignored by MinIO.
+
+If the token is not valid or access is not approved, the plugin must return a `403` (forbidden) HTTP status code. The body must have an `application/json` content-type with the following structure:
+
+```json
+{
+    "reason": <string>
+}
+```
+
+The reason message is returned to the client.
+
+## Example Plugin Implementation
+
+A toy example for the Identity Management Plugin is given [here](./identity-manager-plugin.go).

docs/iam/identity-manager-plugin.go

@@ -0,0 +1,86 @@
+//go:build ignore
+// +build ignore
+
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+)
+
+func writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusBadRequest)
+	json.NewEncoder(w).Encode(map[string]string{
+		"reason": fmt.Sprintf("%v", err),
+	})
+}
+
+type Resp struct {
+	User               string                 `json:"user"`
+	MaxValiditySeconds int                    `json:"maxValiditySeconds"`
+	Claims             map[string]interface{} `json:"claims"`
+}
+
+var tokens map[string]Resp = map[string]Resp{
+	"aaa": {
+		User:               "Alice",
+		MaxValiditySeconds: 3600,
+		Claims: map[string]interface{}{
+			"groups": []string{"data-science"},
+		},
+	},
+	"bbb": {
+		User:               "Bart",
+		MaxValiditySeconds: 3600,
+		Claims: map[string]interface{}{
+			"groups": []string{"databases"},
+		},
+	},
+}
+
+func mainHandler(w http.ResponseWriter, r *http.Request) {
+	token := r.FormValue("token")
+	if token == "" {
+		writeErrorResponse(w, errors.New("token parameter not given"))
+		return
+	}
+
+	rsp, ok := tokens[token]
+	if !ok {
+		w.WriteHeader(http.StatusForbidden)
+		return
+	}
+
+	fmt.Printf("Allowed for token: %s user: %s\n", token, rsp.User)
+
+	w.WriteHeader(http.StatusOK)
+	json.NewEncoder(w).Encode(rsp)
+	return
+}
+
+func main() {
+	http.HandleFunc("/", mainHandler)
+
+	log.Print("Listing on :8081")
+	log.Fatal(http.ListenAndServe(":8081", nil))
+}

docs/metrics/prometheus/grafana/README.md

@@ -11,7 +11,7 @@
 
 Visualize MinIO metrics with our official Grafana dashboard available on the [Grafana dashboard portal](https://grafana.com/grafana/dashboards/13502).
 
-Refer to the dashboard [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/minio-overview.json).
+Refer to the dashboard [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/minio-dashboard.json).
 
 ![Grafana](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/grafana-minio.png)
 

docs/metrics/prometheus/grafana/minio-dashboard.json

@@ -16,22 +16,22 @@
       "name": "Bar gauge",
       "version": ""
     },
-    {
-      "type": "panel",
-      "id": "gauge",
-      "name": "Gauge",
-      "version": ""
-    },
     {
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "8.0.6"
+      "version": "8.2.1"
     },
     {
       "type": "panel",
       "id": "graph",
-      "name": "Graph",
+      "name": "Graph (old)",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "piechart",
+      "name": "Pie chart",
       "version": ""
     },
     {
@@ -56,16 +56,23 @@
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
         "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
         "type": "dashboard"
       }
     ]
   },
-  "description": "MinIO dashboard - https://min.io/",
+  "description": "MinIO Grafana Dashboard - https://min.io/",
   "editable": true,
+  "fiscalYearStartMonth": 0,
   "gnetId": 13502,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1629787190164,
+  "iteration": 1653756543524,
   "links": [
     {
       "icon": "external link",
@@ -77,6 +84,7 @@
       "type": "dashboards"
     }
   ],
+  "liveNow": false,
   "panels": [
     {
       "cacheTimeout": null,
@@ -133,7 +141,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -208,7 +216,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -232,8 +240,19 @@
     {
       "cacheTimeout": null,
       "datasource": "${DS_PROMETHEUS}",
+      "description": "",
       "fieldConfig": {
         "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
           "mappings": [
             {
               "options": {
@@ -245,22 +264,13 @@
               "type": "special"
             }
           ],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "rgba(255, 255, 255, 0.97)",
-                "value": null
-              }
-            ]
-          },
           "unit": "bytes"
         },
         "overrides": []
       },
       "gridPos": {
         "h": 6,
-        "w": 3,
+        "w": 4,
         "x": 6,
         "y": 0
       },
@@ -269,7 +279,15 @@
       "links": [],
       "maxDataPoints": 100,
       "options": {
-        "orientation": "horizontal",
+        "displayLabels": [],
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "values": [
+            "percent"
+          ]
+        },
+        "pieType": "donut",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -277,28 +295,36 @@
           "fields": "",
           "values": false
         },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
+        "tooltip": {
+          "mode": "single"
+        }
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
-          "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance))",
+          "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance))",
           "format": "time_series",
           "instant": false,
           "interval": "1m",
           "intervalFactor": 1,
-          "legendFormat": "",
+          "legendFormat": "Used",
           "refId": "A",
           "step": 300
+        },
+        {
+          "exemplar": true,
+          "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance)) ",
+          "hide": false,
+          "interval": "1m",
+          "legendFormat": "Free",
+          "refId": "B"
         }
       ],
       "timeFrom": null,
       "timeShift": null,
-      "title": "Current Usable Capacity",
-      "type": "gauge"
+      "title": "Capacity",
+      "type": "piechart"
     },
     {
       "aliasColors": {},
@@ -310,8 +336,8 @@
       "fillGradient": 0,
       "gridPos": {
         "h": 6,
-        "w": 7,
-        "x": 9,
+        "w": 6,
+        "x": 10,
         "y": 0
       },
       "hiddenSeries": false,
@@ -332,7 +358,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -434,7 +460,7 @@
         "showUnfilled": false,
         "text": {}
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -512,7 +538,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -588,7 +614,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -668,7 +694,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -748,7 +774,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -828,7 +854,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -911,7 +937,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -962,7 +988,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1058,7 +1084,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1178,7 +1204,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -1258,7 +1284,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -1341,7 +1367,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -1405,7 +1431,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -1473,7 +1499,7 @@
         "text": {},
         "textMode": "auto"
       },
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "targets": [
         {
           "exemplar": true,
@@ -1528,7 +1554,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1624,7 +1650,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1730,7 +1756,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1833,7 +1859,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -1941,7 +1967,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2033,7 +2059,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2125,7 +2151,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2219,7 +2245,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2326,7 +2352,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2443,7 +2469,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2536,7 +2562,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.0.6",
+      "pluginVersion": "8.2.1",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -2606,8 +2632,8 @@
       }
     }
   ],
-  "refresh": "10s",
-  "schemaVersion": 30,
+  "refresh": false,
+  "schemaVersion": 31,
   "style": "dark",
   "tags": [
     "minio"
@@ -2668,7 +2694,7 @@
     ]
   },
   "timezone": "",
-  "title": "MinIO Overview",
+  "title": "MinIO Dashboard",
   "uid": "TgmJnqnnk",
-  "version": 18
+  "version": 7
 }

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-05-19T18-20-59Z
+  image: quay.io/minio/minio:RELEASE.2022-05-26T05-48-41Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/sts/custom-token-identity.go

@@ -0,0 +1,121 @@
+//go:build ignore
+// +build ignore
+
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/url"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	cr "github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var (
+	// LDAP integrated Minio endpoint
+	stsEndpoint string
+
+	// token to use with AssumeRoleWithCustomToken
+	token string
+
+	// Role ARN to use
+	roleArn string
+
+	// Display credentials flag
+	displayCreds bool
+
+	// Credential expiry duration
+	expiryDuration time.Duration
+
+	// Bucket to list
+	bucketToList string
+)
+
+func init() {
+	flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint")
+	flag.StringVar(&token, "t", "", "Token to use with AssumeRoleWithCustomToken STS API (required)")
+	flag.StringVar(&roleArn, "r", "", "RoleARN to use with the request (required)")
+	flag.BoolVar(&displayCreds, "d", false, "Only show generated credentials")
+	flag.DurationVar(&expiryDuration, "e", 0, "Request a duration of validity for the generated credential")
+	flag.StringVar(&bucketToList, "b", "mybucket", "Bucket to list (defaults to mybucket)")
+}
+
+func main() {
+	flag.Parse()
+	if token == "" || roleArn == "" {
+		flag.PrintDefaults()
+		return
+	}
+
+	// The credentials package in minio-go provides an interface to call the
+	// AssumeRoleWithCustomToken STS API.
+
+	var opts []cr.CustomTokenOpt
+	if expiryDuration != 0 {
+		opts = append(opts, cr.CustomTokenValidityOpt(expiryDuration))
+	}
+
+	// Initialize
+	li, err := cr.NewCustomTokenCredentials(stsEndpoint, token, roleArn, opts...)
+	if err != nil {
+		log.Fatalf("Error initializing CustomToken Identity: %v", err)
+	}
+
+	v, err := li.Get()
+	if err != nil {
+		log.Fatalf("Error retrieving STS credentials: %v", err)
+	}
+
+	if displayCreds {
+		fmt.Println("Only displaying credentials:")
+		fmt.Println("AccessKeyID:", v.AccessKeyID)
+		fmt.Println("SecretAccessKey:", v.SecretAccessKey)
+		fmt.Println("SessionToken:", v.SessionToken)
+		return
+	}
+
+	// Use generated credentials to authenticate with MinIO server
+	stsEndpointURL, err := url.Parse(stsEndpoint)
+	if err != nil {
+		log.Fatalf("Error parsing sts endpoint: %v", err)
+	}
+	copts := &minio.Options{
+		Creds:  li,
+		Secure: stsEndpointURL.Scheme == "https",
+	}
+	minioClient, err := minio.New(stsEndpointURL.Host, copts)
+	if err != nil {
+		log.Fatalf("Error initializing client: ", err)
+	}
+
+	// Use minIO Client object normally like the regular client.
+	fmt.Printf("Calling list objects on bucket named `%s` with temp creds:\n===\n", bucketToList)
+	objCh := minioClient.ListObjects(context.Background(), bucketToList, minio.ListObjectsOptions{})
+	for obj := range objCh {
+		if obj.Err != nil {
+			log.Fatalf("Listing error: %v", obj.Err)
+		}
+		fmt.Printf("Key: %s\nSize: %d\nLast Modified: %s\n===\n", obj.Key, obj.Size, obj.LastModified)
+	}
+}

docs/sts/custom-token-identity.md

@@ -0,0 +1,53 @@
+# AssumeRoleWithCustomToken [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
+
+## Introduction
+
+To integrate with custom authentication methods using the [Identity Management Plugin](../iam/identity-management-plugin.md)), MinIO provides an STS API extension called `AssumeRoleWithCustomToken`.
+
+After configuring the plugin, use the generated Role ARN with `AssumeRoleWithCustomToken` to get temporary credentials to access object storage.
+
+## API Request
+
+To make an STS API request with this method, send a POST request to the MinIO endpoint with following query parameters:
+
+| Parameter       | Type    | Required |                                                                      |
+|-----------------|---------|----------|----------------------------------------------------------------------|
+| Action          | String  | Yes      | Value must be `AssumeRoleWithCustomToken`                         |
+| Version         | String  | Yes      | Value must be `2011-06-15`                                           |
+| Token           | String  | Yes      | Token to be authenticated by identity plugin                         |
+| RoleArn         | String  | Yes      | Must match the Role ARN generated for the identity plugin            |
+| DurationSeconds | Integer | No       | Duration of validity of generated credentials. Must be at least 900. |
+
+The validity duration of the generated STS credentials is the minimum of the `DurationSeconds` parameter (if passed) and the validity duration returned by the Identity Management Plugin.
+
+## API Response
+
+XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements)
+
+## Example request and response
+
+Sample request with `curl`:
+
+```sh
+curl -XPOST 'http://localhost:9001/?Action=AssumeRoleWithCustomToken&Version=2011-06-15&Token=aaa&RoleArn=arn:minio:iam:::role/idmp-vGxBdLkOc8mQPU1-UQbBh-yWWVQ'
+```
+
+Prettified Response:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<AssumeRoleWithCustomTokenResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+  <AssumeRoleWithCustomTokenResult>
+    <Credentials>
+      <AccessKeyId>24Y5H9VHE14H47GEOKCX</AccessKeyId>
+      <SecretAccessKey>H+aBfQ9B1AeWWb++84hvp4tlFBo9aP+hUTdLFIeg</SecretAccessKey>
+      <Expiration>2022-05-25T19:56:34Z</Expiration>
+      <SessionToken>eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIyNFk1SDlWSEUxNEg0N0dFT0tDWCIsImV4cCI6MTY1MzUwODU5NCwiZ3JvdXBzIjpbImRhdGEtc2NpZW5jZSJdLCJwYXJlbnQiOiJjdXN0b206QWxpY2UiLCJyb2xlQXJuIjoiYXJuOm1pbmlvOmlhbTo6OnJvbGUvaWRtcC14eHgiLCJzdWIiOiJjdXN0b206QWxpY2UifQ.1tO1LmlUNXiy-wl-ZbkJLWTpaPlhaGqHehsi21lNAmAGCImHHsPb-GA4lRq6GkvHAODN5ZYCf_S-OwpOOdxFwA</SessionToken>
+    </Credentials>
+    <AssumedUser>custom:Alice</AssumedUser>
+  </AssumeRoleWithCustomTokenResult>
+  <ResponseMetadata>
+    <RequestId>16F26E081E36DE63</RequestId>
+  </ResponseMetadata>
+</AssumeRoleWithCustomTokenResponse>
+```

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb v1.0.29
-	github.com/coredns/coredns v1.9.0
+	github.com/coredns/coredns v1.9.2
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dchest/siphash v1.2.1
@@ -21,11 +21,11 @@ require (
 	github.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eclipse/paho.mqtt.golang v1.3.5
-	github.com/elastic/go-elasticsearch/v7 v7.12.0
+	github.com/elastic/go-elasticsearch/v7 v7.17.1
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
 	github.com/fraugster/parquet-go v0.10.0
-	github.com/go-ldap/ldap/v3 v3.2.4
+	github.com/go-ldap/ldap/v3 v3.4.3
 	github.com/go-openapi/loads v0.21.1
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang-jwt/jwt/v4 v4.4.1
@@ -41,7 +41,7 @@ require (
 	github.com/klauspost/readahead v1.4.0
 	github.com/klauspost/reedsolomon v1.9.15
 	github.com/lib/pq v1.10.4
-	github.com/miekg/dns v1.1.46
+	github.com/miekg/dns v1.1.48
 	github.com/minio/cli v1.22.0
 	github.com/minio/console v0.17.2
 	github.com/minio/csvparser v1.0.0
@@ -49,8 +49,8 @@ require (
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.19.2
 	github.com/minio/madmin-go v1.3.14
-	github.com/minio/minio-go/v7 v7.0.26
-	github.com/minio/pkg v1.1.23
+	github.com/minio/minio-go/v7 v7.0.27
+	github.com/minio/pkg v1.1.24
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.4.2
@@ -79,24 +79,24 @@ require (
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yargevad/filepathx v1.0.0
 	github.com/zeebo/xxh3 v1.0.0
-	go.etcd.io/etcd/api/v3 v3.5.2
-	go.etcd.io/etcd/client/v3 v3.5.2
+	go.etcd.io/etcd/api/v3 v3.5.4
+	go.etcd.io/etcd/client/v3 v3.5.4
 	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
+	golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba
 	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
-	google.golang.org/api v0.67.0
+	google.golang.org/api v0.78.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	cloud.google.com/go v0.100.2 // indirect
-	cloud.google.com/go/compute v0.1.0 // indirect
+	cloud.google.com/go/compute v1.6.1 // indirect
 	cloud.google.com/go/iam v0.2.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e // indirect
 	github.com/apache/thrift v0.15.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -118,7 +118,7 @@ require (
 	github.com/frankban/quicktest v1.14.0 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/errors v0.20.2 // indirect
@@ -137,7 +137,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -184,7 +184,7 @@ require (
 	github.com/posener/complete v1.2.3 // indirect
 	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
-	github.com/prometheus/common v0.33.0 // indirect
+	github.com/prometheus/common v0.34.0 // indirect
 	github.com/prometheus/prom2json v1.3.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 // indirect
@@ -201,22 +201,22 @@ require (
 	github.com/unrolled/secure v1.10.0 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
 	go.mongodb.org/mongo-driver v1.8.4 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/goleak v1.1.12 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.8 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a // indirect
-	google.golang.org/grpc v1.44.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect
+	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect

go.sum

@@ -34,8 +34,12 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v0.1.0 h1:rSUBvAyVwNJ5uQCKNJFMwPtTvJkfN38b6Pvb9zZoqJ8=
 cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
+cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
+cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
+cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
+cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc=
+cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
@@ -58,7 +62,7 @@ github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0Sz
 github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE=
+github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ=
@@ -72,8 +76,8 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
-github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
-github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e h1:ZU22z/2YRFLyf/P4ZwUYSdNCWsMEI0VeyrFoI2rAhJQ=
+github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -155,12 +159,13 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/coredns/coredns v1.9.0 h1:M1EF1uups4CYcQGb1z8A97mfoq4BYCw3+xCYcJkOSDc=
-github.com/coredns/coredns v1.9.0/go.mod h1:czzy6Ofs15Mzn1PXpWoplBCZxoWdGoQUInL9uPSiYME=
+github.com/coredns/coredns v1.9.2 h1:r1uPYQ/HKQq8zoQ3NP2V4k1hxb3Yw2xN9AXcXzofh6U=
+github.com/coredns/coredns v1.9.2/go.mod h1:U44W7RM94WPp8soWjsm8g08oOQ1A6D1xu4VyCYJ79cc=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM=
@@ -204,8 +209,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y=
 github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
-github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s=
-github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
+github.com/elastic/go-elasticsearch/v7 v7.17.1 h1:49mHcHx7lpCL8cW1aioEwSEVKQF3s+Igi4Ye/QTWwmk=
+github.com/elastic/go-elasticsearch/v7 v7.17.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -214,6 +219,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -238,8 +244,8 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo
 github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 h1:QqwPZCwh/k1uYqq6uXSb9TRDhTkfQbO80v8zhnIe5zM=
 github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1/go.mod h1:Az6Jt+M5idSED2YPGtwnfJV0kXohgdCBPmHGSYc1r04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=
-github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
+github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -247,8 +253,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
-github.com/go-ldap/ldap/v3 v3.2.4 h1:PFavAq2xTgzo/loE8qNXcQaofAaqIpI4WgaLdv+1l3E=
-github.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg=
+github.com/go-ldap/ldap/v3 v3.4.3 h1:JCKUtJPIcyOuG7ctGabLKMgIlKnGumD/iGjuWeEruDI=
+github.com/go-ldap/ldap/v3 v3.4.3/go.mod h1:7LdHfVt6iIOESVEe3Bs4Jp2sHEKgDeduAhgM1/f9qmo=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
@@ -376,8 +382,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -411,8 +418,10 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
+github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI=
+github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
@@ -595,8 +604,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
-github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ=
+github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ=
 github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
@@ -624,11 +633,11 @@ github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77Z
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
-github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ=
-github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg=
+github.com/minio/minio-go/v7 v7.0.27 h1:yJCvm78B+2+ll1PqO9eSD1as6Ibw3IYnnD8PyBEB2zo=
+github.com/minio/minio-go/v7 v7.0.27/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg=
 github.com/minio/pkg v1.1.20/go.mod h1:Xo7LQshlxGa9shKwJ7NzQbgW4s8T/Wc1cOStR/eUiMY=
-github.com/minio/pkg v1.1.23 h1:CJSoPslQCWZW3z3T79+pv9dVBDCQEK3ipiwXcoAtzY0=
-github.com/minio/pkg v1.1.23/go.mod h1:z9PfmEI804KFkF6eY4LoGe8IDVvTCsYGVuaf58Dr0WI=
+github.com/minio/pkg v1.1.24 h1:a2RCb6LgsCi9DvrripuvlFQRCNb5Hp1HIssnsUqLoZY=
+github.com/minio/pkg v1.1.24/go.mod h1:z9PfmEI804KFkF6eY4LoGe8IDVvTCsYGVuaf58Dr0WI=
 github.com/minio/selfupdate v0.4.0 h1:A7t07pN4Ch1tBTIRStW0KhUVyykz+2muCqFsITQeEW8=
 github.com/minio/selfupdate v0.4.0/go.mod h1:mcDkzMgq8PRcpCRJo/NlPY7U45O5dfYl2Y0Rg7IustY=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -739,6 +748,7 @@ github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -751,8 +761,9 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE=
 github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
+github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE=
+github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -888,14 +899,17 @@ github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+r
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI=
 go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
+go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE=
 go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA=
 go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o=
+go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
+go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
 go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
@@ -934,7 +948,6 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -944,6 +957,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1032,8 +1046,10 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 h1:yssD99+7tqHWO5Gwh81phT+67hg+KttniBr6UnEXOY8=
-golang.org/x/net v0.0.0-20220421235706-1d1ef9303861/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1052,6 +1068,7 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1153,8 +1170,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw=
+golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1247,8 +1268,9 @@ golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1281,8 +1303,13 @@ google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv
 google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
 google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.67.0 h1:lYaaLa+x3VVUhtosaK9xihwQ9H9KRa557REHwwZ2orM=
 google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
+google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
+google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
+google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
+google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
+google.golang.org/api v0.78.0 h1:5ewPyCwP43C3i8B6C2Kb+eVAevbnke2xR8VbcSWjS4I=
+google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1353,8 +1380,17 @@ google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a h1:uqouglH745GoGeZ1YFZbPBiu961tgi/9Qm5jaorajjQ=
-google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
+google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1381,8 +1417,10 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
+google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1396,8 +1434,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=