Update minio-go to v7.0.27

Now returns user does not exist error if the user is not known to the system

.github/workflows/iam-integrations.yaml

@@ -108,6 +108,17 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
+      - name: Test with Access Management Plugin enabled
+        env:
+          LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          ETCD_SERVER: ${{ matrix.etcd }}
+          OPENID_TEST_SERVER: ${{ matrix.openid }}
+          POLICY_PLUGIN_ENDPOINT: "http://127.0.0.1:8080"
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          go run docs/iam/access-manager-plugin.go &
+          make test-iam
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |

README.md

@@ -244,13 +244,13 @@ mc admin update <minio alias, e.g., myminio>
 
 - For deployments without external internet access (e.g. airgapped environments), download the binary from <https://dl.min.io> and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.
 
-- For RPM/DEB installations, upgrade packages **parallelly** on all servers. Once upgraded, perform `systemctl restart minio` across all nodes in **parallel**. RPM/DEB based installations are usually automated using [`ansible`](https://github.com/minio/ansible-minio).
+- For installations using Systemd MinIO service, upgrade via RPM/DEB packages **parallelly** on all servers or replace the binary lets say `/opt/bin/minio` on all nodes, apply executable permissions `chmod +x /opt/bin/minio`. Proceed to perform `systemctl restart minio` across all nodes in **parallel**.
 
 ### Upgrade Checklist
 
 - Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
 - Read the release notes for the targeted MinIO release *before* performing any installation, there is no forced requirement to upgrade to latest releases every week. If it has a bug fix you are looking for then yes, else avoid actively upgrading a running production system.
-- Make sure MinIO process has write access to `/opt/bin` if you plan to use `mc admin update`. This is needed for MinIO to download the latest binary from <https://dl.min.io> and save it locally for upgrades.
+- If you plan to use `mc admin update`, MinIO process must have write access to the parent directory to provide in-place upgrades.
 - `mc admin update` is not supported in kubernetes/container environments, container environments provide their own mechanisms for container updates.
 - **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
 

cmd/admin-handler-utils.go

@@ -180,7 +180,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			apiErr = APIError{
 				Code:           "XMinioAdminTierBackendInUse",
 				Description:    err.Error(),
-				HTTPStatusCode: http.StatusConflict,
+				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case errors.Is(err, errTierBackendNotEmpty):
 			apiErr = APIError{

cmd/admin-handlers-config-kv.go

@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio/internal/config/etcd"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
-	"github.com/minio/minio/internal/config/policy/opa"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -104,7 +104,7 @@ func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config,
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	globalNotificationSys.SignalService(serviceReloadDynamic)
+	globalNotificationSys.SignalConfigReload(subSys)
 	// Tell the client that dynamic config was applied.
 	w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 }
@@ -436,8 +436,8 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 				off = !cache.Enabled(kv)
 			case config.StorageClassSubSys:
 				off = !storageclass.Enabled(kv)
-			case config.PolicyOPASubSys:
-				off = !opa.Enabled(kv)
+			case config.PolicyPluginSubSys:
+				off = !polplugin.Enabled(kv)
 			case config.IdentityOpenIDSubSys:
 				off = !openid.Enabled(kv)
 			case config.IdentityLDAPSubSys:

cmd/admin-handlers-site-replication.go

@@ -241,6 +241,8 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				return
 			}
 		}
+	case madmin.SRBucketMetaTypeVersionConfig:
+		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning)
 	case madmin.SRBucketMetaTypeTags:
 		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
 	case madmin.SRBucketMetaTypeObjectLockConfig:

cmd/admin-handlers-users-race_test.go

@@ -26,6 +26,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -41,6 +42,10 @@ func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
 }
 
 func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip("windows is clunky")
+	}
+
 	baseTestCases := []TestSuiteCommon{
 		// Init and run test on FS backend with signature v4.
 		{serverType: "FS", signer: signerV4},

cmd/admin-handlers-users.go

@@ -677,29 +677,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Call hook for cluster-replication if the service account is not for a
-	// root user.
-	if newCred.ParentUser != globalActiveCred.AccessKey {
-		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-			Type: madmin.SRIAMItemSvcAcc,
-			SvcAccChange: &madmin.SRSvcAccChange{
-				Create: &madmin.SRSvcAccCreate{
-					Parent:        newCred.ParentUser,
-					AccessKey:     newCred.AccessKey,
-					SecretKey:     newCred.SecretKey,
-					Groups:        newCred.Groups,
-					Claims:        opts.claims,
-					SessionPolicy: createReq.Policy,
-					Status:        auth.AccountOn,
-				},
-			},
-		})
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-	}
-
 	createResp := madmin.AddServiceAccountResp{
 		Credentials: madmin.Credentials{
 			AccessKey: newCred.AccessKey,
@@ -720,6 +697,29 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	writeSuccessResponseJSON(w, encryptedData)
+
+	// Call hook for cluster-replication if the service account is not for a
+	// root user.
+	if newCred.ParentUser != globalActiveCred.AccessKey {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Claims:        opts.claims,
+					SessionPolicy: createReq.Policy,
+					Status:        auth.AccountOn,
+				},
+			},
+		})
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return
+		}
+	}
 }
 
 // UpdateServiceAccount - POST /minio/admin/v3/update-service-account
@@ -1105,6 +1105,19 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}
 
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          iampolicy.GetBucketLocationAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          claims,
+		}) {
+			rd = true
+		}
+
 		if globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,

cmd/admin-handlers-users_test.go

@@ -28,6 +28,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -97,6 +98,29 @@ func (s *TestSuiteIAM) iamSetup(c *check) {
 	}
 }
 
+// List of all IAM test suites (i.e. test server configuration combinations)
+// common to tests.
+var iamTestSuites = func() []*TestSuiteIAM {
+	baseTestCases := []TestSuiteCommon{
+		// Init and run test on FS backend with signature v4.
+		{serverType: "FS", signer: signerV4},
+		// Init and run test on FS backend, with tls enabled.
+		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on Erasure backend.
+		{serverType: "Erasure", signer: signerV4},
+		// Init and run test on ErasureSet backend.
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
+	}
+	return testCases
+}()
+
 const (
 	EnvTestEtcdBackend = "ETCD_SERVER"
 )
@@ -166,30 +190,12 @@ func (s *TestSuiteIAM) getUserClient(c *check, accessKey, secretKey, sessionToke
 }
 
 func TestIAMInternalIDPServerSuite(t *testing.T) {
-	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
-		// Init and run test on Erasure backend.
-		{serverType: "Erasure", signer: signerV4},
-		// Init and run test on ErasureSet backend.
-		{serverType: "ErasureSet", signer: signerV4},
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip("windows is clunky disable these tests")
 	}
-	testCases := []*TestSuiteIAM{}
-	for _, bt := range baseTestCases {
-		testCases = append(testCases,
-			newTestSuiteIAM(bt, false),
-			newTestSuiteIAM(bt, true),
-		)
-	}
-	for i, testCase := range testCases {
-		etcdStr := ""
-		if testCase.withEtcdBackend {
-			etcdStr = " (with etcd backend)"
-		}
+	for i, testCase := range iamTestSuites {
 		t.Run(
-			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
 			func(t *testing.T) {
 				suite := testCase
 				c := &check{t, testCase.serverType}
@@ -236,6 +242,7 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	if err != nil {
 		c.Fatalf("unable to set policy: %v", err)
 	}
+
 	client := s.getUserClient(c, accessKey, secretKey, "")
 	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
 	if err != nil {
@@ -973,6 +980,166 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	pluginEndpoint := os.Getenv("POLICY_PLUGIN_ENDPOINT")
+	if pluginEndpoint == "" {
+		c.Skip("POLICY_PLUGIN_ENDPOINT not given - skipping.")
+	}
+
+	configCmds := []string{
+		"policy_plugin",
+		"url=" + pluginEndpoint,
+	}
+
+	_, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " "))
+	if err != nil {
+		c.Fatalf("unable to setup access management plugin for tests: %v", err)
+	}
+
+	s.RestartIAMSuite(c)
+}
+
+// TestIAM_AMPInternalIDPServerSuite - tests for access management plugin
+func TestIAM_AMPInternalIDPServerSuite(t *testing.T) {
+	for i, testCase := range iamTestSuites {
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
+			func(t *testing.T) {
+				suite := testCase
+				c := &check{t, testCase.serverType}
+
+				suite.SetUpSuite(c)
+				defer suite.TearDownSuite(c)
+
+				suite.SetUpAccMgmtPlugin(c)
+
+				suite.TestAccMgmtPlugin(c)
+			},
+		)
+	}
+}
+
+// TestAccMgmtPlugin - this test assumes that the access-management-plugin is
+// the same as the example in `docs/iam/access-manager-plugin.go` -
+// specifically, it denies only `s3:Put*` operations on non-root accounts.
+func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	// 0. Check that owner is able to make-bucket.
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// 1. Create a user.
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	// 2. Check new user appears in listing
+	usersMap, err := s.adm.ListUsers(ctx)
+	if err != nil {
+		c.Fatalf("error listing: %v", err)
+	}
+	v, ok := usersMap[accessKey]
+	if !ok {
+		c.Fatalf("user not listed: %s", accessKey)
+	}
+	c.Assert(v.Status, madmin.AccountEnabled)
+
+	// 3. Check that user is able to make a bucket.
+	client := s.getUserClient(c, accessKey, secretKey, "")
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("user not create bucket: %v", err)
+	}
+
+	// 3.1 check user has access to bucket
+	c.mustListObjects(ctx, client, bucket)
+
+	// 3.2 check that user cannot upload an object.
+	_, err = client.PutObject(ctx, bucket, "objectName", bytes.NewBuffer([]byte("some content")), 12, minio.PutObjectOptions{})
+	if err == nil {
+		c.Fatalf("user was able to upload unexpectedly")
+	}
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  cr.NewStaticV4(accessKey, secretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Create svc acc
+	cr := c.mustCreateSvcAccount(ctx, accessKey, userAdmClient)
+
+	// 1. Check that svc account appears in listing
+	c.assertSvcAccAppearsInListing(ctx, userAdmClient, accessKey, cr.AccessKey)
+
+	// 2. Check that svc account info can be queried
+	c.assertSvcAccInfoQueryable(ctx, userAdmClient, accessKey, cr.AccessKey, false)
+
+	// 3. Check S3 access
+	c.assertSvcAccS3Access(ctx, s, cr, bucket)
+
+	// Check that session policies do not apply - as policy enforcement is
+	// delegated to plugin.
+	{
+		svcAK, svcSK := mustGenerateCredentials(c)
+
+		// This policy does not allow listing objects.
+		policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+		cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+			Policy:     policyBytes,
+			TargetUser: accessKey,
+			AccessKey:  svcAK,
+			SecretKey:  svcSK,
+		})
+		if err != nil {
+			c.Fatalf("Unable to create svc acc: %v", err)
+		}
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		// Though the attached policy does not allow listing, it will be
+		// ignored because the plugin allows it.
+		c.mustListObjects(ctx, svcClient, bucket)
+	}
+
+	// 4. Check that service account's secret key and account status can be
+	// updated.
+	c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket)
+
+	// 5. Check that service account can be deleted.
+	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
+
+	// 6. Check that service account **can** be created for some other user.
+	// This is possible because of the policy enforced in the plugin.
+	c.mustCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
+}
+
 func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
 	randUser := mustGetUUID()
 	randPass := mustGetUUID()
@@ -1026,7 +1193,7 @@ func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bu
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
 	v, ok := <-res
 	if !ok || v.Err == nil {
-		c.Fatalf("user was able to list unexpectedly!")
+		c.Fatalf("user was able to list unexpectedly! on %s", bucket)
 	}
 }
 
@@ -1034,11 +1201,20 @@ func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucke
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
 	v, ok := <-res
 	if ok && v.Err != nil {
-		msg := fmt.Sprintf("user was unable to list: %v", v.Err)
-		c.Fatalf(msg)
+		c.Fatalf("user was unable to list: %v", v.Err)
 	}
 }
 
+func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
+	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
+	if e, ok := err.(minio.ErrorResponse); ok {
+		if e.Code == "AccessDenied" {
+			return
+		}
+	}
+	c.Fatalf("upload did not get an AccessDenied error - got %#v instead", err)
+}
+
 func (c *check) assertSvcAccS3Access(ctx context.Context, s *TestSuiteIAM, cr madmin.Credentials, bucket string) {
 	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
 	c.mustListObjects(ctx, svcClient, bucket)

cmd/admin-handlers.go

@@ -57,7 +57,8 @@ import (
 )
 
 const (
-	maxEConfigJSONSize = 262272
+	maxEConfigJSONSize        = 262272
+	kubernetesVersionEndpoint = "https://kubernetes.default.svc/version"
 )
 
 // Only valid query params for mgmt admin APIs.
@@ -653,12 +654,10 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	globalProfilerMu.Unlock()
 
 	timer := time.NewTimer(duration)
+	defer timer.Stop()
 	for {
 		select {
 		case <-ctx.Done():
-			if !timer.Stop() {
-				<-timer.C
-			}
 			for k, v := range globalProfiler {
 				v.Stop()
 				delete(globalProfiler, k)
@@ -1768,6 +1767,36 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 	}
 }
 
+func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
+	ctx, cancel := context.WithCancel(dctx)
+	defer cancel()
+
+	ki := madmin.KubernetesInfo{}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, kubernetesVersionEndpoint, nil)
+	if err != nil {
+		ki.Error = err.Error()
+		return ki
+	}
+
+	client := &http.Client{
+		Transport: NewGatewayHTTPTransport(),
+		Timeout:   10 * time.Second,
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		ki.Error = err.Error()
+		return ki
+	}
+
+	decoder := json.NewDecoder(resp.Body)
+	if err := decoder.Decode(&ki); err != nil {
+		ki.Error = err.Error()
+	}
+	return ki
+}
+
 // HealthInfoHandler - GET /minio/admin/v3/healthinfo
 // ----------
 // Get server health info
@@ -1855,6 +1884,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		info.SetAddr(anonAddr(info.GetAddr()))
 	}
 
+	getAndWritePlatformInfo := func() {
+		if IsKubernetes() {
+			healthInfo.Sys.KubernetesInfo = getKubernetesInfo(deadlinedCtx)
+			partialWrite(healthInfo)
+		}
+	}
+
 	getAndWriteCPUs := func() {
 		if query.Get("syscpu") == "true" {
 			localCPUInfo := madmin.GetCPUs(deadlinedCtx, globalLocalNodeName)
@@ -2177,6 +2213,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		defer close(healthInfoCh)
 
 		partialWrite(healthInfo) // Write first message with only version and deployment id populated
+		getAndWritePlatformInfo()
 		getAndWriteCPUs()
 		getAndWritePartitions()
 		getAndWriteOSInfo()

cmd/admin-heal-ops.go

@@ -194,7 +194,6 @@ func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	for {
 		select {
 		case <-periodicTimer.C:
-			periodicTimer.Reset(time.Minute * 5)
 			now := UTCNow()
 			ahs.Lock()
 			for path, h := range ahs.healSeqMap {
@@ -203,6 +202,8 @@ func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 				}
 			}
 			ahs.Unlock()
+
+			periodicTimer.Reset(time.Minute * 5)
 		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately
@@ -581,12 +582,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	// heal-results in memory and the client has not consumed it
 	// for too long.
 	unconsumedTimer := time.NewTimer(healUnconsumedTimeout)
-	defer func() {
-		// stop the timeout timer so it is garbage collected.
-		if !unconsumedTimer.Stop() {
-			<-unconsumedTimer.C
-		}
-	}()
+	defer unconsumedTimer.Stop()
 
 	var itemsLen int
 	for {

cmd/api-errors.go

@@ -280,6 +280,7 @@ const (
 	ErrSiteReplicationBucketConfigError
 	ErrSiteReplicationBucketMetaError
 	ErrSiteReplicationIAMError
+	ErrSiteReplicationConfigMissing
 
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
@@ -900,7 +901,7 @@ var errorCodes = errorCodeMap{
 	ErrReplicationDenyEditError: {
 		Code:           "XMinioReplicationDenyEdit",
 		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
@@ -1166,7 +1167,7 @@ var errorCodes = errorCodeMap{
 	ErrObjectExistsAsDirectory: {
 		Code:           "XMinioObjectExistsAsDirectory",
 		Description:    "Object name already exists as a directory.",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidObjectName: {
 		Code:           "XMinioInvalidObjectName",
@@ -1340,7 +1341,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Error while replicating an IAM item",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-
+	ErrSiteReplicationConfigMissing: {
+		Code:           "XMinioSiteReplicationConfigMissingError",
+		Description:    "Site not found in site replication configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",

cmd/auth-handler.go

@@ -217,8 +217,8 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		}
 	}
 
-	// If OPA is set, return without any further checks.
-	if globalPolicyOPA != nil {
+	// If AuthZPlugin is set, return without any further checks.
+	if newGlobalAuthZPluginFn() != nil {
 		return claims.Map(), nil
 	}
 

cmd/background-newdisks-heal-ops.go

@@ -312,9 +312,6 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 		case <-ctx.Done():
 			return
 		case <-diskCheckTimer.C:
-			// Reset to next interval.
-			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
-
 			var erasureSetInPoolDisksToHeal []map[int][]StorageAPI
 
 			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
@@ -448,6 +445,9 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 				}
 			}
 			wg.Wait()
+
+			// Reset for next interval.
+			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 		}
 	}
 }

cmd/bucket-handlers.go

@@ -364,6 +364,18 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			}) {
 				bucketsInfo[n] = bucketInfo
 				n++
+			} else if globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
+				Action:          iampolicy.GetBucketLocationAction,
+				BucketName:      bucketInfo.Name,
+				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				IsOwner:         owner,
+				ObjectName:      "",
+				Claims:          cred.Claims,
+			}) {
+				bucketsInfo[n] = bucketInfo
+				n++
 			}
 		}
 		bucketsInfo = bucketsInfo[:n]
@@ -471,9 +483,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		hasLockEnabled = true
 	}
 
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	suspended := globalBucketVersioningSys.Suspended(bucket)
-
 	type deleteResult struct {
 		delInfo DeletedObject
 		errInfo DeleteError
@@ -481,8 +490,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
 
+	vc, _ := globalBucketVersioningSys.Get(bucket)
 	oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
-
 	for index, object := range deleteObjectsReq.Objects {
 		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
 			if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
@@ -514,8 +523,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 		opts := ObjectOptions{
 			VersionID:        object.VersionID,
-			Versioned:        versioned,
-			VersionSuspended: suspended,
+			Versioned:        vc.PrefixEnabled(object.ObjectName),
+			VersionSuspended: vc.Suspended(),
 		}
 
 		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
@@ -526,7 +535,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if !globalTierConfigMgr.Empty() {
-			oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(opts.VersionID).WithVersioning(versioned, suspended)
+			oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
 			oss[index].SetTransitionState(goi.TransitionedObject)
 		}
 
@@ -581,8 +590,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		Versioned:        versioned,
-		VersionSuspended: suspended,
+		PrefixEnabledFn:  vc.PrefixEnabled,
+		VersionSuspended: vc.Suspended(),
 	})
 
 	for i := range errs {
@@ -638,22 +647,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
-		if replicateDeletes {
-			if dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending {
-				dv := DeletedObjectReplicationInfo{
-					DeletedObject: dobj,
-					Bucket:        bucket,
-				}
-				scheduleReplicationDelete(ctx, dv, objectAPI)
+		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
+			dv := DeletedObjectReplicationInfo{
+				DeletedObject: dobj,
+				Bucket:        bucket,
 			}
-		}
-
-	}
-
-	// Notify deleted event for objects.
-	for _, dobj := range deletedObjects {
-		if dobj.ObjectName == "" {
-			continue
+			scheduleReplicationDelete(ctx, dv, objectAPI)
 		}
 
 		eventName := event.ObjectRemovedDelete
@@ -1625,10 +1624,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if globalSiteReplicationSys.isEnabled() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
-		return
-	}
+
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return

cmd/bucket-lifecycle.go

@@ -330,7 +330,7 @@ const (
 // 2. when a transitioned object expires (based on an ILM rule).
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
 	var opts ObjectOptions
-	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
+	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
 	opts.Expiration = ExpirationOptions{Expire: true}
 	switch action {
@@ -415,8 +415,8 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 			ETag:   oi.ETag,
 		},
 		VersionID:        oi.VersionID,
-		Versioned:        globalBucketVersioningSys.Enabled(oi.Bucket),
-		VersionSuspended: globalBucketVersioningSys.Suspended(oi.Bucket),
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		MTime:            oi.ModTime,
 	}
 	return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
@@ -575,8 +575,8 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 
 // postRestoreOpts returns ObjectOptions with version-id from the POST restore object request for a given bucket and object.
 func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
+	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 	vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
@@ -627,8 +627,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
-			Versioned:        globalBucketVersioningSys.Enabled(bucket),
-			VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+			Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
+			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
 			UserDefined:      meta,
 		}
 	}
@@ -640,8 +640,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	}
 
 	return ObjectOptions{
-		Versioned:        globalBucketVersioningSys.Enabled(bucket),
-		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
 		UserDefined:      meta,
 		VersionID:        objInfo.VersionID,
 		MTime:            objInfo.ModTime,

cmd/bucket-metadata-sys.go

@@ -196,7 +196,7 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
-		return nil, err
+		return &versioning.Versioning{}, err
 	}
 	return meta.versioningConfig, nil
 }

cmd/bucket-replication-stats.go

@@ -135,6 +135,23 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	return st.Clone()
 }
 
+// GetAll returns replication metrics for all buckets at once.
+func (r *ReplicationStats) GetAll() map[string]BucketReplicationStats {
+	if r == nil {
+		return map[string]BucketReplicationStats{}
+	}
+
+	r.RLock()
+	defer r.RUnlock()
+
+	bucketReplicationStats := make(map[string]BucketReplicationStats, len(r.Cache))
+	for k, v := range r.Cache {
+		bucketReplicationStats[k] = v.Clone()
+	}
+
+	return bucketReplicationStats
+}
+
 // Get replication metrics for a bucket from this node since this node came up.
 func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	if r == nil {
@@ -161,7 +178,7 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 
 // load replication metrics at cluster start from initial data usage
 func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
-	rTimer := time.NewTimer(time.Minute * 1)
+	rTimer := time.NewTimer(time.Minute)
 	defer rTimer.Stop()
 	var (
 		dui DataUsageInfo
@@ -174,13 +191,12 @@ outer:
 			return
 		case <-rTimer.C:
 			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
-			if err != nil {
-				continue
-			}
 			// If LastUpdate is set, data usage is available.
-			if !dui.LastUpdate.IsZero() {
+			if err == nil && !dui.LastUpdate.IsZero() {
 				break outer
 			}
+
+			rTimer.Reset(time.Minute)
 		}
 	}
 

cmd/bucket-replication-utils.go

@@ -509,7 +509,7 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 				ObjectName: oi.Name,
 				VersionID:  oi.VersionID,
 			},
-		}, oi, ObjectOptions{}, nil)
+		}, oi, ObjectOptions{VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)}, nil)
 	} else {
 		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
 			UserDefined: oi.UserDefined,

cmd/bucket-replication.go

@@ -180,6 +180,11 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica
 		return
 	}
 
+	// Disable server-side replication on object prefixes which are excluded
+	// from versioning via the MinIO bucket versioning extension.
+	if globalBucketVersioningSys.PrefixSuspended(bucket, object) {
+		return
+	}
 	replStatus := mopts.ReplicationStatus()
 	if replStatus == replication.Replica && !mopts.isMetadataReplication() {
 		return
@@ -264,6 +269,11 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 	if delOpts.ReplicationRequest {
 		return
 	}
+	// Skip replication if this object's prefix is excluded from being
+	// versioned.
+	if !delOpts.Versioned {
+		return
+	}
 	opts := replication.ObjectOpts{
 		Name:         dobj.ObjectName,
 		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
@@ -457,8 +467,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		VersionID:         versionID,
 		MTime:             dobj.DeleteMarkerMTime.Time,
 		DeleteReplication: drs,
-		Versioned:         globalBucketVersioningSys.Enabled(bucket),
-		VersionSuspended:  globalBucketVersioningSys.Suspended(bucket),
+		Versioned:         globalBucketVersioningSys.PrefixEnabled(bucket, dobj.ObjectName),
+		VersionSuspended:  globalBucketVersioningSys.PrefixSuspended(bucket, dobj.ObjectName),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
 		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
@@ -1590,7 +1600,11 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 	return reader, proxyResult{Proxy: true}, nil
 }
 
-func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) {
+func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) {
+	if opts.VersionSuspended {
+		return &madmin.BucketTargets{}
+	}
+
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || cfg == nil {
 		return &madmin.BucketTargets{}
@@ -1831,9 +1845,26 @@ func resyncTarget(oi ObjectInfo, arn string, resetID string, resetBeforeDate tim
 	return
 }
 
-// get the most current of in-memory replication stats  and data usage info from crawler.
-func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
-	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
+func getAllLatestReplicationStats(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
+	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
+	bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
+
+	for bucket, u := range bucketsUsage {
+		bucketStats := make([]BucketStats, len(peerBucketStatsList))
+		for i, peerBucketStats := range peerBucketStatsList {
+			bucketStat, ok := peerBucketStats[bucket]
+			if !ok {
+				continue
+			}
+			bucketStats[i] = bucketStat
+		}
+		bucketsReplicationStats[bucket] = calculateBucketReplicationStats(bucket, u, bucketStats)
+	}
+
+	return bucketsReplicationStats
+}
+
+func calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
 	// accumulate cluster bucket stats
 	stats := make(map[string]*BucketReplicationStat)
 	var totReplicaSize int64
@@ -1902,6 +1933,12 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	return s
 }
 
+// get the most current of in-memory replication stats  and data usage info from crawler.
+func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
+	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
+	return calculateBucketReplicationStats(bucket, u, bucketStats)
+}
+
 const resyncTimeInterval = time.Minute * 10
 
 // periodicResyncMetaSave saves in-memory resync meta stats to disk in periodic intervals
@@ -1912,7 +1949,6 @@ func (p *ReplicationPool) periodicResyncMetaSave(ctx context.Context, objectAPI
 	for {
 		select {
 		case <-resyncTimer.C:
-			resyncTimer.Reset(resyncTimeInterval)
 			now := UTCNow()
 			p.resyncState.RLock()
 			for bucket, brs := range p.resyncState.statusMap {
@@ -1933,6 +1969,8 @@ func (p *ReplicationPool) periodicResyncMetaSave(ctx context.Context, objectAPI
 				}
 			}
 			p.resyncState.RUnlock()
+
+			resyncTimer.Reset(resyncTimeInterval)
 		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately

cmd/bucket-stats.go

@@ -51,6 +51,9 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 	rl.UploadHistogram.Add(size, duration)
 }
 
+// BucketStatsMap captures bucket statistics for all buckets
+type BucketStatsMap map[string]BucketStats
+
 // BucketStats bucket statistics
 type BucketStats struct {
 	ReplicationStats BucketReplicationStats

cmd/bucket-stats_gen.go

@@ -792,6 +792,183 @@ func (z *BucketStats) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *BucketStatsMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0003 uint32
+	zb0003, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if (*z) == nil {
+		(*z) = make(BucketStatsMap, zb0003)
+	} else if len((*z)) > 0 {
+		for key := range *z {
+			delete((*z), key)
+		}
+	}
+	for zb0003 > 0 {
+		zb0003--
+		var zb0001 string
+		var zb0002 BucketStats
+		zb0001, err = dc.ReadString()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		var field []byte
+		_ = field
+		var zb0004 uint32
+		zb0004, err = dc.ReadMapHeader()
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return
+		}
+		for zb0004 > 0 {
+			zb0004--
+			field, err = dc.ReadMapKeyPtr()
+			if err != nil {
+				err = msgp.WrapError(err, zb0001)
+				return
+			}
+			switch msgp.UnsafeString(field) {
+			case "ReplicationStats":
+				err = zb0002.ReplicationStats.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, zb0001, "ReplicationStats")
+					return
+				}
+			default:
+				err = dc.Skip()
+				if err != nil {
+					err = msgp.WrapError(err, zb0001)
+					return
+				}
+			}
+		}
+		(*z)[zb0001] = zb0002
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteMapHeader(uint32(len(z)))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0005, zb0006 := range z {
+		err = en.WriteString(zb0005)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		// map header, size 1
+		// write "ReplicationStats"
+		err = en.Append(0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
+		if err != nil {
+			return
+		}
+		err = zb0006.ReplicationStats.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, zb0005, "ReplicationStats")
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendMapHeader(o, uint32(len(z)))
+	for zb0005, zb0006 := range z {
+		o = msgp.AppendString(o, zb0005)
+		// map header, size 1
+		// string "ReplicationStats"
+		o = append(o, 0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
+		o, err = zb0006.ReplicationStats.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, zb0005, "ReplicationStats")
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketStatsMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0003 uint32
+	zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if (*z) == nil {
+		(*z) = make(BucketStatsMap, zb0003)
+	} else if len((*z)) > 0 {
+		for key := range *z {
+			delete((*z), key)
+		}
+	}
+	for zb0003 > 0 {
+		var zb0001 string
+		var zb0002 BucketStats
+		zb0003--
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		var field []byte
+		_ = field
+		var zb0004 uint32
+		zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return
+		}
+		for zb0004 > 0 {
+			zb0004--
+			field, bts, err = msgp.ReadMapKeyZC(bts)
+			if err != nil {
+				err = msgp.WrapError(err, zb0001)
+				return
+			}
+			switch msgp.UnsafeString(field) {
+			case "ReplicationStats":
+				bts, err = zb0002.ReplicationStats.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, zb0001, "ReplicationStats")
+					return
+				}
+			default:
+				bts, err = msgp.Skip(bts)
+				if err != nil {
+					err = msgp.WrapError(err, zb0001)
+					return
+				}
+			}
+		}
+		(*z)[zb0001] = zb0002
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BucketStatsMap) Msgsize() (s int) {
+	s = msgp.MapHeaderSize
+	if z != nil {
+		for zb0005, zb0006 := range z {
+			_ = zb0006
+			s += msgp.StringPrefixSize + len(zb0005) + 1 + 17 + zb0006.ReplicationStats.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *ReplicationLatency) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-stats_gen_test.go

@@ -348,6 +348,119 @@ func BenchmarkDecodeBucketStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalBucketStatsMap(t *testing.T) {
+	v := BucketStatsMap{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketStatsMap(t *testing.T) {
+	v := BucketStatsMap{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketStatsMap Msgsize() is inaccurate")
+	}
+
+	vn := BucketStatsMap{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicationLatency(t *testing.T) {
 	v := ReplicationLatency{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-targets.go

@@ -201,7 +201,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
 		if err == nil {
-			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{}) {
+			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{OpType: replication.AllReplicationType}) {
 				if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) {
 					sys.RLock()
 					_, ok := sys.arnRemotesMap[arnStr]

cmd/bucket-versioning-handler.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/xml"
 	"io"
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
@@ -63,28 +65,28 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if globalSiteReplicationSys.isEnabled() {
+	if globalSiteReplicationSys.isEnabled() && !v.Enabled() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
-			Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
-			HTTPStatusCode: http.StatusConflict,
+			Description:    "Cluster replication is enabled on this site, versioning cannot be suspended on bucket.",
+			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL)
 		return
 	}
 
-	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
+	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && (v.Suspended() || v.PrefixesExcluded()) {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
-			Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
-			HTTPStatusCode: http.StatusConflict,
+			Description:    "An Object Lock configuration is present on this bucket, versioning cannot be suspended.",
+			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL)
 		return
 	}
 	if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
-			Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
-			HTTPStatusCode: http.StatusConflict,
+			Description:    "A replication configuration is present on this bucket, bucket wide versioning cannot be suspended.",
+			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL)
 		return
 	}
@@ -100,6 +102,20 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:       madmin.SRBucketMetaTypeVersionConfig,
+		Bucket:     bucket,
+		Versioning: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	writeSuccessResponseHeadersOnly(w)
 }
 

cmd/bucket-versioning.go

@@ -31,6 +31,17 @@ func (sys *BucketVersioningSys) Enabled(bucket string) bool {
 	return vc.Enabled()
 }
 
+// PrefixEnabled returns true is versioning is enabled at bucket level and if
+// the given prefix doesn't match any excluded prefixes pattern. This is
+// part of a MinIO versioning configuration extension.
+func (sys *BucketVersioningSys) PrefixEnabled(bucket, prefix string) bool {
+	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	if err != nil {
+		return false
+	}
+	return vc.PrefixEnabled(prefix)
+}
+
 // Suspended suspended versioning?
 func (sys *BucketVersioningSys) Suspended(bucket string) bool {
 	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
@@ -40,15 +51,19 @@ func (sys *BucketVersioningSys) Suspended(bucket string) bool {
 	return vc.Suspended()
 }
 
+// PrefixSuspended returns true if the given prefix matches an excluded prefix
+// pattern. This is part of a MinIO versioning configuration extension.
+func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
+	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	if err != nil {
+		return false
+	}
+
+	return vc.PrefixSuspended(prefix)
+}
+
 // Get returns stored bucket policy
 func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-		return nil, NotImplemented{}
-	}
 	return globalBucketMetadataSys.GetVersioningConfig(bucket)
 }
 

cmd/common-main.go

@@ -190,6 +190,10 @@ func minioConfigToConsoleFeatures() {
 		os.Setenv("CONSOLE_PROMETHEUS_URL", value)
 		if value := env.Get("MINIO_PROMETHEUS_JOB_ID", "minio-job"); value != "" {
 			os.Setenv("CONSOLE_PROMETHEUS_JOB_ID", value)
+			// Support additional labels for more granular filtering.
+			if value := env.Get("MINIO_PROMETHEUS_EXTRA_LABELS", ""); value != "" {
+				os.Setenv("CONSOLE_PROMETHEUS_EXTRA_LABELS", value)
+			}
 		}
 	}
 	// Enable if LDAP is enabled.

cmd/config-current.go

@@ -37,6 +37,7 @@ import (
 	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/scanner"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/config/subnet"
@@ -56,6 +57,7 @@ func initHelp() {
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
 		config.IdentityTLSSubSys:    xtls.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
+		config.PolicyPluginSubSys:   polplugin.DefaultKVS,
 		config.SiteSubSys:           config.DefaultSiteKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
@@ -107,8 +109,8 @@ func initHelp() {
 			Description: "enable X.509 TLS certificate SSO support",
 		},
 		config.HelpKV{
-			Key:         config.PolicyOPASubSys,
-			Description: "[DEPRECATED] enable external OPA for policy enforcement",
+			Key:         config.PolicyPluginSubSys,
+			Description: "enable Access Management Plugin for policy enforcement",
 		},
 		config.HelpKV{
 			Key:         config.APISubSys,
@@ -219,6 +221,7 @@ func initHelp() {
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.IdentityTLSSubSys:    xtls.Help,
 		config.PolicyOPASubSys:      opa.Help,
+		config.PolicyPluginSubSys:   polplugin.Help,
 		config.LoggerWebhookSubSys:  logger.Help,
 		config.AuditWebhookSubSys:   logger.HelpWebhook,
 		config.AuditKafkaSubSys:     logger.HelpKafka,
@@ -243,6 +246,10 @@ func initHelp() {
 			Key:         config.RegionSubSys,
 			Description: "[DEPRECATED - use `site` instead] label the location of the server",
 		},
+		config.PolicyOPASubSys: {
+			Key:         config.PolicyOPASubSys,
+			Description: "[DEPRECATED - use `policy_plugin` instead] enable external OPA for policy enforcement",
+		},
 	}
 
 	config.RegisterHelpDeprecatedSubSys(deprecatedHelpKVMap)
@@ -340,9 +347,20 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	case config.PolicyOPASubSys:
-		if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+		// In case legacy OPA config is being set, we treat it as if the
+		// AuthZPlugin is being set.
+		subSys = config.PolicyPluginSubSys
+		fallthrough
+	case config.PolicyPluginSubSys:
+		if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
 			NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
 			return err
+		} else if ppargs.URL == nil {
+			// Check if legacy opa is configured.
+			if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+				NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+				return err
+			}
 		}
 	default:
 		if config.LoggerSubSystems.Contains(subSys) {
@@ -523,12 +541,25 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
 
-	opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+	authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OPA: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
 	}
-	globalPolicyOPA = opa.New(opaCfg)
+	if authZPluginCfg.URL == nil {
+		opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+			NewGatewayHTTPTransport(), xhttp.DrainBody)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
+		} else {
+			authZPluginCfg.URL = opaCfg.URL
+			authZPluginCfg.AuthToken = opaCfg.AuthToken
+			authZPluginCfg.Transport = opaCfg.Transport
+			authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
+		}
+	}
+
+	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
 
 	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
 		globalRootCAs)
@@ -609,7 +640,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.HTTP {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceIf
+				l.LogOnce = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 				loggerCfg.HTTP[n] = l
@@ -627,7 +658,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.AuditWebhook {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceIf
+				l.LogOnce = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 				loggerCfg.AuditWebhook[n] = l

cmd/data-scanner.go

@@ -197,9 +197,6 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 		case <-ctx.Done():
 			return
 		case <-scannerTimer.C:
-			// Reset the timer for next cycle.
-			scannerTimer.Reset(scannerCycle.Get())
-
 			if intDataUpdateTracker.debug {
 				console.Debugln("starting scanner cycle")
 			}
@@ -232,6 +229,9 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 					logger.LogIf(ctx, err)
 				}
 			}
+
+			// Reset the timer for next cycle.
+			scannerTimer.Reset(scannerCycle.Get())
 		}
 	}
 }
@@ -1184,7 +1184,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		opts.VersionID = obj.VersionID
 	}
 	if opts.VersionID == "" {
-		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
+		opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
+		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 	}
 
 	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
@@ -1243,6 +1244,9 @@ func (i *scannerItem) objectPath() string {
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
 	roi := getHealReplicateObjectInfo(oi, i.replication)
+	if !roi.Dsc.ReplicateAny() {
+		return
+	}
 
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		// heal delete marker replication failure or versioned delete replication failure

cmd/disk-cache-backend.go

@@ -884,6 +884,7 @@ func (c *diskCache) put(ctx context.Context, bucket, object string, data io.Read
 	}
 
 	if err := os.MkdirAll(cachePath, 0o777); err != nil {
+		removeAll(cachePath)
 		return oi, err
 	}
 	metadata := cloneMSS(opts.UserDefined)
@@ -892,6 +893,7 @@ func (c *diskCache) put(ctx context.Context, bucket, object string, data io.Read
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
+			removeAll(cachePath)
 			return oi, err
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(size))
@@ -1628,8 +1630,6 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-timer.C:
-			// Reset for the next interval
-			timer.Reset(cacheStaleUploadCleanupInterval)
 			now := time.Now()
 			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir), func(shaDir string, typ os.FileMode) error {
 				return readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir), func(uploadIDDir string, typ os.FileMode) error {
@@ -1660,6 +1660,9 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 				}
 				return nil
 			})
+
+			// Reset for the next interval
+			timer.Reset(cacheStaleUploadCleanupInterval)
 		}
 	}
 }

cmd/disk-cache.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/wildcard"
@@ -739,35 +738,30 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
 	infoCh := make(chan ObjectInfo)
-	errorCh := make(chan error)
 	go func() {
+		defer close(infoCh)
 		info, err := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader), opts)
-		if err != nil {
-			close(infoCh)
-			pipeReader.CloseWithError(err)
-			rPipe.CloseWithError(err)
-			errorCh <- err
-			return
+		pipeReader.CloseWithError(err)
+		rPipe.CloseWithError(err)
+		if err == nil {
+			infoCh <- info
 		}
-		close(errorCh)
-		infoCh <- info
 	}()
 
 	go func() {
 		_, err := dcache.put(lkctx.Context(), bucket, object, rPipe, r.Size(), nil, opts, false, false)
 		if err != nil {
-			rPipe.CloseWithError(err)
-			return
+			logger.LogIf(lkctx.Context(), err)
 		}
+		// We do not care about errors to cached backend.
+		rPipe.Close()
 	}()
 
 	mwriter := cacheMultiWriter(pipeWriter, wPipe)
 	_, err = io.Copy(mwriter, r)
 	pipeWriter.Close()
 	wPipe.Close()
-
 	if err != nil {
-		err = <-errorCh
 		return ObjectInfo{}, err
 	}
 	info := <-infoCh
@@ -803,8 +797,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 		return
 	}
 	var opts ObjectOptions
-	opts.UserDefined = make(map[string]string)
-	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
+	opts.UserDefined = cloneMSS(oi.UserDefined)
 	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader), opts)
 	wbCommitStatus := CommitComplete
 	size := objInfo.Size

cmd/erasure-healing.go

@@ -418,10 +418,19 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		return result, nil
 	}
 
+	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+		// When disk to heal count is greater than parity blocks we should simply error out.
+		err := fmt.Errorf("more disks are expected to heal than parity, returned errors: %v -> %s/%s(%s)", errs, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	cleanFileInfo := func(fi FileInfo) FileInfo {
-		// Returns a copy of the 'fi' with checksums and parts nil'ed.
+		// Returns a copy of the 'fi' with erasure index, checksums and inline data niled.
 		nfi := fi
-		if !fi.IsRemote() {
+		if !nfi.IsRemote() {
+			nfi.Data = nil
 			nfi.Erasure.Index = 0
 			nfi.Erasure.Checksums = nil
 		}
@@ -432,12 +441,25 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	tmpID := mustGetUUID()
 	migrateDataDir := mustGetUUID()
 
+	// Reorder so that we have data disks first and parity disks next.
+	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
+	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
+
 	copyPartsMetadata := make([]FileInfo, len(partsMetadata))
+	for i := range latestDisks {
+		if latestDisks[i] == nil {
+			continue
+		}
+		copyPartsMetadata[i] = partsMetadata[i]
+	}
+
 	for i := range outDatedDisks {
 		if outDatedDisks[i] == nil {
 			continue
 		}
-		copyPartsMetadata[i] = partsMetadata[i]
+		// Make sure to write the FileInfo information
+		// that is expected to be in quorum.
 		partsMetadata[i] = cleanFileInfo(latestMeta)
 	}
 
@@ -456,12 +478,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 	}
 
-	// Reorder so that we have data disks first and parity disks next.
-	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
-	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
-	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
-	copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
-
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
 		// Heal each part. erasureHealFile() will write the healed
 		// part to .minio/tmp/uuid/ which needs to be renamed later to
@@ -486,7 +502,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 				checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber)
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
-				readers[i] = newBitrotReader(disk, partsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
+				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
 					checksumInfo.Hash, erasure.ShardSize())
 			}
 			writers := make([]io.Writer, len(outDatedDisks))

cmd/erasure-metadata-utils.go

@@ -178,10 +178,6 @@ func shuffleDisksAndPartsMetadataByIndex(disks []StorageAPI, metaArr []FileInfo,
 			inconsistent++
 			continue
 		}
-		if len(fi.Data) != len(meta.Data) {
-			inconsistent++
-			continue
-		}
 		if meta.XLV1 != fi.XLV1 {
 			inconsistent++
 			continue
@@ -229,12 +225,6 @@ func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo,
 			// if object was ever written previously.
 			continue
 		}
-		if !init && len(fi.Data) != len(partsMetadata[index].Data) {
-			// Check for length of data parts only when
-			// fi.ModTime is not empty - ModTime is always set,
-			// if object was ever written previously.
-			continue
-		}
 		if !init && fi.XLV1 != partsMetadata[index].XLV1 {
 			continue
 		}

cmd/erasure-metadata.go

@@ -107,7 +107,7 @@ func (fi FileInfo) IsValid() bool {
 func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	object = decodeDirObject(object)
 	versionID := fi.VersionID
-	if (globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket)) && versionID == "" {
+	if (globalBucketVersioningSys.PrefixEnabled(bucket, object) || globalBucketVersioningSys.PrefixSuspended(bucket, object)) && versionID == "" {
 		versionID = nullVersionID
 	}
 
@@ -297,8 +297,6 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 				fmt.Fprintf(h, "part.%d", part.Number)
 			}
 			fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
-			// make sure that length of Data is same
-			fmt.Fprintf(h, "%v", len(meta.Data))
 
 			// ILM transition fields
 			fmt.Fprint(h, meta.TransitionStatus)

cmd/erasure-multipart.go

@@ -190,6 +190,8 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 // towards simplification of multipart APIs.
 // The resulting ListMultipartsInfo structure is unmarshalled directly as XML.
 func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	result.MaxUploads = maxUploads
 	result.KeyMarker = keyMarker
 	result.Prefix = object
@@ -284,8 +286,10 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 // disks. `uploads.json` carries metadata regarding on-going multipart
 // operation(s) on the object.
 func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, object string, opts ObjectOptions) (string, error) {
+	userDefined := cloneMSS(opts.UserDefined)
+
 	onlineDisks := er.getDisks()
-	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
+	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
 	if parityDrives <= 0 {
 		parityDrives = er.defaultParityCount
 	}
@@ -306,7 +310,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		}
 	}
 	if parityOrig != parityDrives {
-		opts.UserDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 	}
 
 	dataDrives := len(onlineDisks) - parityDrives
@@ -334,8 +338,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Guess content-type from the extension if possible.
-	if opts.UserDefined["content-type"] == "" {
-		opts.UserDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
+	if userDefined["content-type"] == "" {
+		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
 	modTime := opts.MTime
@@ -350,7 +354,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	for index := range partsMetadata {
 		partsMetadata[index].Fresh = true
 		partsMetadata[index].ModTime = modTime
-		partsMetadata[index].Metadata = opts.UserDefined
+		partsMetadata[index].Metadata = userDefined
 	}
 
 	uploadID := mustGetUUID()
@@ -371,10 +375,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 //
 // Implements S3 compatible initiate multipart API.
 func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (string, error) {
-	// No metadata is set, allocate a new one.
-	if opts.UserDefined == nil {
-		opts.UserDefined = make(map[string]string)
-	}
+	auditObjectErasureSet(ctx, object, &er)
+
 	return er.newMultipartUpload(ctx, bucket, object, opts)
 }
 
@@ -446,6 +448,8 @@ func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, ds
 //
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	// Write lock for this part ID.
 	// Held throughout the operation.
 	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
@@ -683,6 +687,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 // - encrypted
 // - compressed
 func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	result := MultipartInfo{
 		Bucket:   bucket,
 		Object:   object,
@@ -739,6 +745,8 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
@@ -832,6 +840,8 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 //
 // Implements S3 compatible Complete multipart API.
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	// Hold read-locks to verify uploaded parts, also disallows
 	// parallel part uploads as well.
 	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
@@ -1028,6 +1038,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // would be removed from the system, rollback is not possible on this
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {

cmd/erasure-object.go

@@ -63,6 +63,8 @@ func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
 // if source object and destination object are same we only
 // update metadata.
 func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
+	auditObjectErasureSet(ctx, dstObject, &er)
+
 	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
 	if !srcInfo.metadataOnly {
 		return oi, NotImplemented{}
@@ -131,15 +133,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		modTime = UTCNow()
 	}
 
-	// If the data is not inlined, we may end up incorrectly
-	// inlining the data here, that leads to an inconsistent
-	// situation where some objects are were not inlined
-	// were now inlined, make sure to `nil` the Data such
-	// that xl.meta is written as expected.
-	if !fi.InlineData() {
-		fi.Data = nil
-	}
-
 	fi.VersionID = versionID // set any new versionID we might have created
 	fi.ModTime = modTime     // set modTime for the new versionID
 	if !dstOpts.MTime.IsZero() {
@@ -149,12 +142,15 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	fi.Metadata = srcInfo.UserDefined
 	srcInfo.UserDefined["etag"] = srcInfo.ETag
 
+	inlineData := fi.InlineData()
+	freeVersionID := fi.TierFreeVersionID()
+	freeVersionMarker := fi.TierFreeVersion()
+
 	// Update `xl.meta` content on each disks.
 	for index := range metaArr {
 		if metaArr[index].IsValid() {
 			metaArr[index].ModTime = modTime
 			metaArr[index].VersionID = versionID
-			metaArr[index].Metadata = srcInfo.UserDefined
 			if !metaArr[index].InlineData() {
 				// If the data is not inlined, we may end up incorrectly
 				// inlining the data here, that leads to an inconsistent
@@ -163,6 +159,17 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 				// that xl.meta is written as expected.
 				metaArr[index].Data = nil
 			}
+			metaArr[index].Metadata = srcInfo.UserDefined
+			// Preserve existing values
+			if inlineData {
+				metaArr[index].SetInlineData()
+			}
+			if freeVersionID != "" {
+				metaArr[index].SetTierFreeVersionID(freeVersionID)
+			}
+			if freeVersionMarker {
+				metaArr[index].SetTierFreeVersion()
+			}
 		}
 	}
 
@@ -177,6 +184,8 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // GetObjectNInfo - returns object info and an object
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
 	defer func() {
@@ -413,6 +422,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	if !opts.NoLock {
 		// Lock the object before reading.
 		lk := er.NewNSLock(bucket, object)
@@ -832,19 +843,18 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	data := r.Reader
 
-	// No metadata is set, allocate a new one.
-	if opts.UserDefined == nil {
-		opts.UserDefined = make(map[string]string)
-	}
+	userDefined := cloneMSS(opts.UserDefined)
 
 	storageDisks := er.getDisks()
 
 	parityDrives := len(storageDisks) / 2
 	if !opts.MaxParity {
 		// Get parity and data drive count based on storage class metadata
-		parityDrives = globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
+		parityDrives = globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
 		if parityDrives <= 0 {
 			parityDrives = er.defaultParityCount
 		}
@@ -882,7 +892,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			parityDrives = len(storageDisks) / 2
 		}
 		if parityOrig != parityDrives {
-			opts.UserDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+			userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 		}
 	}
 	dataDrives := len(storageDisks) - parityDrives
@@ -1058,13 +1068,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			Hash:       bitrotWriterSum(w),
 		})
 	}
-	if opts.UserDefined["etag"] == "" {
-		opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	if userDefined["etag"] == "" {
+		userDefined["etag"] = r.MD5CurrentHexString()
 	}
 
 	// Guess content-type from the extension if possible.
-	if opts.UserDefined["content-type"] == "" {
-		opts.UserDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
+	if userDefined["content-type"] == "" {
+		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
 	modTime := opts.MTime
@@ -1075,7 +1085,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
-		partsMetadata[index].Metadata = opts.UserDefined
+		partsMetadata[index].Metadata = userDefined
 		partsMetadata[index].Size = n
 		partsMetadata[index].ModTime = modTime
 	}
@@ -1148,6 +1158,10 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	for _, obj := range objects {
+		auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+	}
+
 	errs := make([]error, len(objects))
 	dobjects := make([]DeletedObject, len(objects))
 	writeQuorums := make([]int, len(objects))
@@ -1178,7 +1192,13 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		// VersionID is not set means delete is not specific about
 		// any version, look for if the bucket is versioned or not.
 		if objects[i].VersionID == "" {
-			if opts.Versioned || opts.VersionSuspended {
+			// MinIO extension to bucket version configuration
+			suspended := opts.VersionSuspended
+			versioned := opts.Versioned
+			if opts.PrefixEnabledFn != nil {
+				versioned = opts.PrefixEnabledFn(objects[i].ObjectName)
+			}
+			if versioned || suspended {
 				// Bucket is versioned and no version was explicitly
 				// mentioned for deletes, create a delete marker instead.
 				vr.ModTime = UTCNow()
@@ -1186,7 +1206,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 				// Versioning suspended means that we add a `null` version
 				// delete marker, if not add a new version for this delete
 				// marker.
-				if opts.Versioned {
+				if versioned {
 					vr.VersionID = mustGetUUID()
 				}
 			}
@@ -1328,6 +1348,8 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	auditObjectErasureSet(ctx, object, &er)
+
 	if opts.DeletePrefix {
 		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
 	}

cmd/erasure-object_test.go

@@ -20,11 +20,14 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	crand "crypto/rand"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"testing"
 
@@ -1048,3 +1051,63 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		})
 	}
 }
+
+// In some deployments, one object has data inlined in one disk and not inlined in other disks.
+func TestGetObjectInlineNotInline(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create a backend with 4 disks named disk{1...4}, this name convention
+	// because we will unzip some object data from a sample archive.
+	const numDisks = 4
+	path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var fsDirs []string
+	for i := 1; i <= numDisks; i++ {
+		fsDirs = append(fsDirs, filepath.Join(path, fmt.Sprintf("disk%d", i)))
+	}
+
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		removeRoots(fsDirs)
+		t.Fatal(err)
+	}
+
+	// cleaning up of temporary test directories
+	defer objLayer.Shutdown(context.Background())
+	defer removeRoots(fsDirs)
+
+	// Create a testbucket
+	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", BucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unzip sample object data to the existing disks
+	err = unzipArchive("testdata/xl-meta-inline-notinline.zip", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Try to read the object and check its md5sum
+	gr, err := objLayer.GetObjectNInfo(ctx, "testbucket", "file", nil, nil, readLock, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
+	}
+
+	h := md5.New()
+	_, err = io.Copy(h, gr)
+	if err != nil {
+		t.Fatalf("Expected GetObject reading data to succeed, but failed with %v", err)
+	}
+	gr.Close()
+
+	const expectedHash = "fffb6377948ebea75ad2b8058e849ef5"
+	foundHash := fmt.Sprintf("%x", h.Sum(nil))
+	if foundHash != expectedHash {
+		t.Fatalf("Expected data to have md5sum = `%s`, found `%s`", expectedHash, foundHash)
+	}
+}

cmd/erasure-server-pool-decom.go

@@ -541,8 +541,13 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 }
 
 func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
-	defer gr.Close()
 	objInfo := gr.ObjInfo
+
+	defer func() {
+		gr.Close()
+		auditLogDecom(ctx, "DecomCopyData", objInfo.Bucket, objInfo.Name, objInfo.VersionID, err)
+	}()
+
 	if objInfo.isMultipart() {
 		uploadID, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
@@ -603,6 +608,8 @@ func (v versionsSorter) reverse() {
 }
 
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
+	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
+
 	var wg sync.WaitGroup
 	wStr := env.Get("_MINIO_DECOMMISSION_WORKERS", strconv.Itoa(len(pool.sets)))
 	workerSize, err := strconv.Atoi(wStr)
@@ -612,7 +619,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	parallelWorkers := make(chan struct{}, workerSize)
 
-	versioned := globalBucketVersioningSys.Enabled(bName)
 	for _, set := range pool.sets {
 		set := set
 		disks := set.getOnlineDisks()
@@ -622,6 +628,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			continue
 		}
 
+		vc, _ := globalBucketVersioningSys.Get(bName)
 		decommissionEntry := func(entry metaCacheEntry) {
 			defer func() {
 				<-parallelWorkers
@@ -660,7 +667,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						bName,
 						version.Name,
 						ObjectOptions{
-							Versioned:         versioned,
+							Versioned:         vc.PrefixEnabled(version.Name),
 							VersionID:         version.VersionID,
 							MTime:             version.ModTime,
 							DeleteReplication: version.ReplicationState,
@@ -680,27 +687,34 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					continue
 				}
 
-				gr, err := set.GetObjectNInfo(ctx,
-					bName,
-					encodeDirObject(version.Name),
-					nil,
-					http.Header{},
-					noLock, // all mutations are blocked reads are safe without locks.
-					ObjectOptions{
-						VersionID: version.VersionID,
-					})
-				if err != nil {
-					logger.LogIf(ctx, err)
-					z.poolMetaMutex.Lock()
-					z.poolMeta.CountItem(idx, version.Size, true)
-					z.poolMetaMutex.Unlock()
-					break // break out on first error
-				}
 				var failure bool
 				// gr.Close() is ensured by decommissionObject().
-				if err = z.decommissionObject(ctx, bName, gr); err != nil {
-					logger.LogIf(ctx, err)
-					failure = true
+				for try := 0; try < 3; try++ {
+					gr, err := set.GetObjectNInfo(ctx,
+						bName,
+						encodeDirObject(version.Name),
+						nil,
+						http.Header{},
+						noLock, // all mutations are blocked reads are safe without locks.
+						ObjectOptions{
+							VersionID: version.VersionID,
+						})
+					if isErrObjectNotFound(err) {
+						// object deleted by the application, nothing to do here we move on.
+						return
+					}
+					if err != nil {
+						failure = true
+						logger.LogIf(ctx, err)
+						continue
+					}
+					if err = z.decommissionObject(ctx, bName, gr); err != nil {
+						failure = true
+						logger.LogIf(ctx, err)
+						continue
+					}
+					failure = false
+					break
 				}
 				z.poolMetaMutex.Lock()
 				z.poolMeta.CountItem(idx, version.Size, failure)
@@ -713,13 +727,17 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 			// if all versions were decommissioned, then we can delete the object versions.
 			if decommissionedCount == len(fivs.Versions) {
-				set.DeleteObject(ctx,
+				_, err := set.DeleteObject(ctx,
 					bName,
-					entry.name,
+					encodeDirObject(entry.name),
 					ObjectOptions{
 						DeletePrefix: true, // use prefix delete to delete all versions at once.
 					},
 				)
+				auditLogDecom(ctx, "DecomDeleteObject", bName, entry.name, "", err)
+				if err != nil {
+					logger.LogIf(ctx, err)
+				}
 			}
 			z.poolMetaMutex.Lock()
 			z.poolMeta.TrackCurrentBucketObject(idx, bName, entry.name)
@@ -804,6 +822,9 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	dctx, z.decommissionCancelers[idx] = context.WithCancel(GlobalContext)
 	z.poolMetaMutex.Unlock()
 
+	// Generate an empty request info so it can be directly modified later by audit
+	dctx = logger.SetReqInfo(dctx, &logger.ReqInfo{})
+
 	if err := z.decommissionInBackground(dctx, idx); err != nil {
 		logger.LogIf(GlobalContext, err)
 		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
@@ -1075,3 +1096,16 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 	globalNotificationSys.ReloadPoolMeta(ctx)
 	return nil
 }
+
+func auditLogDecom(ctx context.Context, apiName, bucket, object, versionID string, err error) {
+	errStr := ""
+	if err != nil {
+		errStr = err.Error()
+	}
+	auditLogInternal(ctx, bucket, object, AuditLogOptions{
+		Trigger:   "decommissioning",
+		APIName:   apiName,
+		VersionID: versionID,
+		Error:     errStr,
+	})
+}

cmd/erasure-server-pool.go

@@ -233,8 +233,9 @@ func (z *erasureServerPools) SetDriveCounts() []int {
 type serverPoolsAvailableSpace []poolAvailableSpace
 
 type poolAvailableSpace struct {
-	Index     int
-	Available uint64
+	Index      int
+	Available  uint64
+	MaxUsedPct int // Used disk percentage of most filled disk, rounded down.
 }
 
 // TotalAvailable - total available space
@@ -246,10 +247,41 @@ func (p serverPoolsAvailableSpace) TotalAvailable() uint64 {
 	return total
 }
 
+// FilterMaxUsed will filter out any pools that has used percent bigger than max,
+// unless all have that, in which case all are preserved.
+func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
+	// We aren't modifying p, only entries in it, so we don't need to receive a pointer.
+	if len(p) <= 1 {
+		// Nothing to do.
+		return
+	}
+	var ok bool
+	for _, z := range p {
+		if z.MaxUsedPct < max {
+			ok = true
+			break
+		}
+	}
+	if !ok {
+		// All above limit.
+		// Do not modify
+		return
+	}
+
+	// Remove entries that are above.
+	for i, z := range p {
+		if z.MaxUsedPct < max {
+			continue
+		}
+		p[i].Available = 0
+	}
+}
+
 // getAvailablePoolIdx will return an index that can hold size bytes.
 // -1 is returned if no serverPools have available space for the size given.
 func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, object string, size int64) int {
 	serverPools := z.getServerPoolsAvailableSpace(ctx, bucket, object, size)
+	serverPools.FilterMaxUsed(100 - (100 * diskReserveFraction))
 	total := serverPools.TotalAvailable()
 	if total == 0 {
 		return -1
@@ -270,11 +302,13 @@ func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, ob
 
 // getServerPoolsAvailableSpace will return the available space of each pool after storing the content.
 // If there is not enough space the pool will return 0 bytes available.
+// The size of each will be multiplied by the number of sets.
 // Negative sizes are seen as 0 bytes.
 func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, bucket, object string, size int64) serverPoolsAvailableSpace {
 	serverPools := make(serverPoolsAvailableSpace, len(z.serverPools))
 
 	storageInfos := make([][]*DiskInfo, len(z.serverPools))
+	nSets := make([]int, len(z.serverPools))
 	g := errgroup.WithNErrs(len(z.serverPools))
 	for index := range z.serverPools {
 		index := index
@@ -282,9 +316,11 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 		if z.IsSuspended(index) {
 			continue
 		}
+		pool := z.serverPools[index]
+		nSets[index] = pool.setCount
 		g.Go(func() error {
 			// Get the set where it would be placed.
-			storageInfos[index] = getDiskInfos(ctx, z.serverPools[index].getHashedSet(object).getDisks())
+			storageInfos[index] = getDiskInfos(ctx, pool.getHashedSet(object).getDisks())
 			return nil
 		}, index)
 	}
@@ -298,15 +334,30 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 			serverPools[i] = poolAvailableSpace{Index: i}
 			continue
 		}
+		var maxUsedPct int
 		for _, disk := range zinfo {
 			if disk == nil {
 				continue
 			}
 			available += disk.Total - disk.Used
+
+			// set maxUsedPct to the value from the disk with the least space percentage.
+			if pctUsed := int(disk.Used * 100 / disk.Total); pctUsed > maxUsedPct {
+				maxUsedPct = pctUsed
+			}
 		}
+
+		// Since we are comparing pools that may have a different number of sets
+		// we multiply by the number of sets in the pool.
+		// This will compensate for differences in set sizes
+		// when choosing destination pool.
+		// Different set sizes are already compensated by less disks.
+		available *= uint64(nSets[i])
+
 		serverPools[i] = poolAvailableSpace{
-			Index:     i,
-			Available: available,
+			Index:      i,
+			Available:  available,
+			MaxUsedPct: maxUsedPct,
 		}
 	}
 	return serverPools
@@ -774,7 +825,16 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
 	}
 
 	lockType = noLock // do not take locks at lower levels for GetObjectNInfo()
-	return z.serverPools[zIdx].GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	gr, err = z.serverPools[zIdx].GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	if unlockOnDefer {
+		unlockOnDefer = false
+		return gr.WithCleanupFuncs(nsUnlocker), nil
+	}
+	return gr, nil
 }
 
 // getLatestObjectInfoWithIdx returns the objectInfo of the latest object from multiple pools (this function

cmd/erasure-sets.go

@@ -301,14 +301,14 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 		case <-ctx.Done():
 			return
 		case <-monitor.C:
-			// Reset the timer once fired for required interval.
-			monitor.Reset(monitorInterval)
-
 			if serverDebugLog {
 				console.Debugln("running disk monitoring")
 			}
 
 			s.connectDisks()
+
+			// Reset the timer for next interval
+			monitor.Reset(monitorInterval)
 		}
 	}
 }
@@ -485,7 +485,9 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 	go s.cleanupDeletedObjects(ctx)
 
 	// Start the disk monitoring and connect routine.
-	go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
+	if !globalIsTesting {
+		go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
+	}
 
 	return s, nil
 }
@@ -502,9 +504,6 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-timer.C:
-			// Reset for the next interval
-			timer.Reset(globalAPIConfig.getDeleteCleanupInterval())
-
 			var wg sync.WaitGroup
 			for _, set := range s.sets {
 				wg.Add(1)
@@ -517,6 +516,9 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context) {
 				}(set)
 			}
 			wg.Wait()
+
+			// Reset for the next interval
+			timer.Reset(globalAPIConfig.getDeleteCleanupInterval())
 		}
 	}
 }
@@ -530,9 +532,6 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-timer.C:
-			// Reset for the next interval
-			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
-
 			var wg sync.WaitGroup
 			for _, set := range s.sets {
 				wg.Add(1)
@@ -545,6 +544,9 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 				}(set)
 			}
 			wg.Wait()
+
+			// Reset for the next interval
+			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
 		}
 	}
 }
@@ -572,6 +574,7 @@ func (a *auditObjectErasureMap) MarshalJSON() ([]byte, error) {
 	return json.Marshal(mapCopy)
 }
 
+// Add erasure set information to the current context
 func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjects) {
 	if len(logger.AuditTargets()) == 0 {
 		return
@@ -933,21 +936,18 @@ func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, er
 // GetObjectNInfo - returns object info and locked object ReadCloser
 func (s *erasureSets) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 }
 
 // PutObject - writes an object to hashedSet based on the object name.
 func (s *erasureSets) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.PutObject(ctx, bucket, object, data, opts)
 }
 
 // GetObjectInfo - reads object metadata from the hashedSet based on the object name.
 func (s *erasureSets) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.GetObjectInfo(ctx, bucket, object, opts)
 }
 
@@ -967,13 +967,11 @@ func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix st
 
 // DeleteObject - deletes an object from the hashedSet based on the object name.
 func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
-
 	if opts.DeletePrefix {
 		err := s.deletePrefix(ctx, bucket, object)
 		return ObjectInfo{}, err
 	}
+	set := s.getHashedSet(object)
 	return set.DeleteObject(ctx, bucket, object, opts)
 }
 
@@ -1030,9 +1028,6 @@ func (s *erasureSets) DeleteObjects(ctx context.Context, bucket string, objects
 			for i, obj := range group {
 				delErrs[obj.origIndex] = errs[i]
 				delObjects[obj.origIndex] = dobjects[i]
-				if errs[i] == nil {
-					auditObjectErasureSet(ctx, obj.object.ObjectName, set)
-				}
 			}
 		}(s.sets[setIdx], objsGroup)
 	}
@@ -1046,8 +1041,6 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 	srcSet := s.getHashedSet(srcObject)
 	dstSet := s.getHashedSet(dstObject)
 
-	auditObjectErasureSet(ctx, dstObject, dstSet)
-
 	cpSrcDstSame := srcSet == dstSet
 	// Check if this request is only metadata update.
 	if cpSrcDstSame && srcInfo.metadataOnly {
@@ -1086,14 +1079,12 @@ func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix,
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
 	set := s.getHashedSet(prefix)
-	auditObjectErasureSet(ctx, prefix, set)
 	return set.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 }
 
 // Initiate a new multipart upload on a hashedSet based on object name.
 func (s *erasureSets) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.NewMultipartUpload(ctx, bucket, object, opts)
 }
 
@@ -1102,42 +1093,36 @@ func (s *erasureSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject,
 	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions,
 ) (partInfo PartInfo, err error) {
 	destSet := s.getHashedSet(destObject)
-	auditObjectErasureSet(ctx, destObject, destSet)
 	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)
 }
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.
 func (s *erasureSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 }
 
 // GetMultipartInfo - return multipart metadata info uploaded at hashedSet.
 func (s *erasureSets) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (result MultipartInfo, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 }
 
 // ListObjectParts - lists all uploaded parts to an object in hashedSet.
 func (s *erasureSets) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
 func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 }
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.
 func (s *erasureSets) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	set := s.getHashedSet(object)
-	auditObjectErasureSet(ctx, object, set)
 	return set.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 }
 

cmd/fs-v1-multipart.go

@@ -887,8 +887,6 @@ func (fs *FSObjects) cleanupStaleUploads(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-bgAppendTmpCleaner.C:
-			bgAppendTmpCleaner.Reset(bgAppendsCleanupInterval)
-
 			foundUploadIDs := fs.getAllUploadIDs(ctx)
 
 			// Remove background append map from the memory
@@ -915,10 +913,8 @@ func (fs *FSObjects) cleanupStaleUploads(ctx context.Context) {
 				}
 			}
 
+			bgAppendTmpCleaner.Reset(bgAppendsCleanupInterval)
 		case <-expiryUploadsTimer.C:
-			// Reset for the next interval
-			expiryUploadsTimer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
-
 			expiry := globalAPIConfig.getStaleUploadsExpiry()
 			now := time.Now()
 
@@ -944,6 +940,9 @@ func (fs *FSObjects) cleanupStaleUploads(ctx context.Context) {
 					fs.appendFileMapMu.Unlock()
 				}
 			}
+
+			// Reset for the next interval
+			expiryUploadsTimer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
 		}
 	}
 }

cmd/gateway/s3/gateway-s3-sse.go

@@ -364,6 +364,7 @@ func (l *s3EncObjects) GetObjectInfo(ctx context.Context, bucket string, object
 // CopyObject copies an object from source bucket to a destination bucket.
 func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, s, d minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	cpSrcDstSame := path.Join(srcBucket, srcObject) == path.Join(dstBucket, dstObject)
+	userDefined := minio.CloneMSS(srcInfo.UserDefined)
 	if cpSrcDstSame {
 		var gwMeta gwMetaV1
 		if s.ServerSideEncryption != nil && d.ServerSideEncryption != nil &&
@@ -378,16 +379,16 @@ func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObje
 				d.ServerSideEncryption.Marshal(header)
 			}
 			for k, v := range header {
-				srcInfo.UserDefined[k] = v[0]
+				userDefined[k] = v[0]
 			}
-			gwMeta.Meta = srcInfo.UserDefined
+			gwMeta.Meta = userDefined
 			if err = l.writeGWMetadata(ctx, dstBucket, getDareMetaPath(dstObject), gwMeta, minio.ObjectOptions{}); err != nil {
 				return objInfo, minio.ErrorRespToObjectError(err)
 			}
 			return gwMeta.ToObjectInfo(dstBucket, dstObject), nil
 		}
 	}
-	dstOpts := minio.ObjectOptions{ServerSideEncryption: d.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
+	dstOpts := minio.ObjectOptions{ServerSideEncryption: d.ServerSideEncryption, UserDefined: userDefined}
 	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, dstOpts)
 }
 

cmd/gateway/s3/gateway-s3.go

@@ -502,17 +502,20 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 // PutObject creates a new object with the incoming data,
 func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
+
+	userDefined := minio.CloneMSS(opts.UserDefined)
+
 	var tagMap map[string]string
-	if tagstr, ok := opts.UserDefined[xhttp.AmzObjectTagging]; ok && tagstr != "" {
+	if tagstr, ok := userDefined[xhttp.AmzObjectTagging]; ok && tagstr != "" {
 		tagObj, err := tags.ParseObjectTags(tagstr)
 		if err != nil {
 			return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 		}
 		tagMap = tagObj.ToMap()
-		delete(opts.UserDefined, xhttp.AmzObjectTagging)
+		delete(userDefined, xhttp.AmzObjectTagging)
 	}
 	putOpts := miniogo.PutObjectOptions{
-		UserMetadata:         opts.UserDefined,
+		UserMetadata:         userDefined,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		UserTags:             tagMap,
 		// Content-Md5 is needed for buckets with object locking,
@@ -529,7 +532,7 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 		ETag:     ui.ETag,
 		Size:     ui.Size,
 		Key:      object,
-		Metadata: minio.ToMinioClientObjectInfoMetadata(opts.UserDefined),
+		Metadata: minio.ToMinioClientObjectInfoMetadata(userDefined),
 	}
 
 	return minio.FromMinioClientObjectInfo(bucket, oi), nil
@@ -544,8 +547,9 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
 	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
 	// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
-	srcInfo.UserDefined["x-amz-metadata-directive"] = "REPLACE"
-	srcInfo.UserDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
+	userDefined := minio.CloneMSS(srcInfo.UserDefined)
+	userDefined["x-amz-metadata-directive"] = "REPLACE"
+	userDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
 	header := make(http.Header)
 	if srcOpts.ServerSideEncryption != nil {
 		encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
@@ -556,10 +560,10 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 	}
 
 	for k, v := range header {
-		srcInfo.UserDefined[k] = v[0]
+		userDefined[k] = v[0]
 	}
 
-	if _, err = l.Client.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined, miniogo.CopySrcOptions{}, miniogo.PutObjectOptions{}); err != nil {
+	if _, err = l.Client.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, userDefined, miniogo.CopySrcOptions{}, miniogo.PutObjectOptions{}); err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
 	}
 	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)
@@ -605,17 +609,18 @@ func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 // NewMultipartUpload upload object in multiple parts
 func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (uploadID string, err error) {
 	var tagMap map[string]string
-	if tagStr, ok := o.UserDefined[xhttp.AmzObjectTagging]; ok {
+	userDefined := minio.CloneMSS(o.UserDefined)
+	if tagStr, ok := userDefined[xhttp.AmzObjectTagging]; ok {
 		tagObj, err := tags.Parse(tagStr, true)
 		if err != nil {
 			return uploadID, minio.ErrorRespToObjectError(err, bucket, object)
 		}
 		tagMap = tagObj.ToMap()
-		delete(o.UserDefined, xhttp.AmzObjectTagging)
+		delete(userDefined, xhttp.AmzObjectTagging)
 	}
 	// Create PutObject options
 	opts := miniogo.PutObjectOptions{
-		UserMetadata:         o.UserDefined,
+		UserMetadata:         userDefined,
 		ServerSideEncryption: o.ServerSideEncryption,
 		UserTags:             tagMap,
 	}
@@ -645,9 +650,8 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
 	if srcOpts.CheckPrecondFn != nil && srcOpts.CheckPrecondFn(srcInfo) {
 		return minio.PartInfo{}, minio.PreConditionFailed{}
 	}
-	srcInfo.UserDefined = map[string]string{
-		"x-amz-copy-source-if-match": srcInfo.ETag,
-	}
+	userDefined := minio.CloneMSS(srcInfo.UserDefined)
+	userDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
 	header := make(http.Header)
 	if srcOpts.ServerSideEncryption != nil {
 		encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
@@ -657,11 +661,11 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
 		dstOpts.ServerSideEncryption.Marshal(header)
 	}
 	for k, v := range header {
-		srcInfo.UserDefined[k] = v[0]
+		userDefined[k] = v[0]
 	}
 
 	completePart, err := l.Client.CopyObjectPart(ctx, srcBucket, srcObject, destBucket, destObject,
-		uploadID, partID, startOffset, length, srcInfo.UserDefined)
+		uploadID, partID, startOffset, length, userDefined)
 	if err != nil {
 		return p, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
 	}

cmd/globals.go

@@ -42,7 +42,7 @@ import (
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
-	"github.com/minio/minio/internal/config/policy/opa"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/config/subnet"
 	xhttp "github.com/minio/minio/internal/http"
@@ -113,6 +113,10 @@ const (
 	// diskFillFraction is the fraction of a disk we allow to be filled.
 	diskFillFraction = 0.99
 
+	// diskReserveFraction is the fraction of a disk where we will fill other server pools first.
+	// If all pools reach this, we will use all pools with regular placement.
+	diskReserveFraction = 0.15
+
 	// diskAssumeUnknownSize is the size to assume when an unknown size upload is requested.
 	diskAssumeUnknownSize = 1 << 30
 
@@ -139,6 +143,9 @@ var (
 	// Indicates if the running minio is in gateway mode.
 	globalIsGateway = false
 
+	// Indicates if server code should go through testing path.
+	globalIsTesting = false
+
 	// Name of gateway server, e.g S3, NAS etc
 	globalGatewayName = ""
 
@@ -290,8 +297,8 @@ var (
 	// Some standard content-types which we strictly dis-allow for compression.
 	standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
 
-	// OPA policy system.
-	globalPolicyOPA *opa.Opa
+	// AuthZ Plugin system.
+	globalAuthZPlugin *polplugin.AuthZPlugin
 
 	// Deployment ID - unique per deployment
 	globalDeploymentID string
@@ -352,4 +359,18 @@ var (
 	// Add new variable global values here.
 )
 
+var globalAuthZPluginMutex sync.Mutex
+
+func newGlobalAuthZPluginFn() *polplugin.AuthZPlugin {
+	globalAuthZPluginMutex.Lock()
+	defer globalAuthZPluginMutex.Unlock()
+	return globalAuthZPlugin
+}
+
+func setGlobalAuthZPlugin(authz *polplugin.AuthZPlugin) {
+	globalAuthZPluginMutex.Lock()
+	globalAuthZPlugin = authz
+	globalAuthZPluginMutex.Unlock()
+}
+
 var errSelfTestFailure = errors.New("self test failed. unsafe to start server")

cmd/iam-store.go

@@ -244,6 +244,8 @@ type iamWatchEvent struct {
 
 // iamCache contains in-memory cache of IAM data.
 type iamCache struct {
+	updatedAt time.Time
+
 	// map of policy names to policy definitions
 	iamPolicyDocsMap map[string]PolicyDoc
 	// map of usernames to credentials
@@ -432,8 +434,7 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
 func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 	newCache := newIamCache()
 
-	cache := store.lock()
-	defer store.unlock()
+	loadedAt := time.Now()
 
 	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
 		err := iamOS.loadAllFromObjStore(ctx, newCache)
@@ -486,12 +487,28 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 		newCache.buildUserGroupMemberships()
 	}
 
-	cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap
-	cache.iamGroupsMap = newCache.iamGroupsMap
-	cache.iamPolicyDocsMap = newCache.iamPolicyDocsMap
-	cache.iamUserGroupMemberships = newCache.iamUserGroupMemberships
-	cache.iamUserPolicyMap = newCache.iamUserPolicyMap
-	cache.iamUsersMap = newCache.iamUsersMap
+	cache := store.lock()
+	defer store.unlock()
+
+	// We should only update the in-memory cache if there were no changes
+	// to the in-memory cache since the disk loading began. If there
+	// were changes to the in-memory cache we should wait for the next
+	// cycle until we can safely update the in-memory cache.
+	//
+	// An in-memory cache must be replaced only if we know for sure that
+	// the values loaded from disk are not stale. They might be stale
+	// if the cached.updatedAt is recent than the refresh cycle began.
+	if cache.updatedAt.Before(loadedAt) {
+		// No one has updated anything since the config was loaded,
+		// so we just replace whatever is on the disk into memory.
+		cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap
+		cache.iamGroupsMap = newCache.iamGroupsMap
+		cache.iamPolicyDocsMap = newCache.iamPolicyDocsMap
+		cache.iamUserGroupMemberships = newCache.iamUserGroupMemberships
+		cache.iamUserPolicyMap = newCache.iamUserPolicyMap
+		cache.iamUsersMap = newCache.iamUsersMap
+		cache.updatedAt = time.Now()
+	}
 
 	return nil
 }
@@ -548,6 +565,8 @@ func (store *IAMStoreSys) GroupNotificationHandler(ctx context.Context, group st
 		cache.removeGroupFromMembershipsMap(group)
 		delete(cache.iamGroupsMap, group)
 		delete(cache.iamGroupPolicyMap, group)
+
+		cache.updatedAt = time.Now()
 		return nil
 	}
 
@@ -562,6 +581,7 @@ func (store *IAMStoreSys) GroupNotificationHandler(ctx context.Context, group st
 	// removed, the cache stays current.
 	cache.removeGroupFromMembershipsMap(group)
 	cache.updateGroupMembershipsMap(group, &gi)
+	cache.updatedAt = time.Now()
 	return nil
 }
 
@@ -639,6 +659,8 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 		cache.iamUserGroupMemberships[member] = gset
 	}
 
+	cache.updatedAt = time.Now()
+
 	return nil
 }
 
@@ -672,6 +694,7 @@ func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamC
 		cache.iamUserGroupMemberships[member] = gset
 	}
 
+	cache.updatedAt = time.Now()
 	return nil
 }
 
@@ -720,6 +743,7 @@ func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string
 		// Delete from server memory
 		delete(cache.iamGroupsMap, group)
 		delete(cache.iamGroupPolicyMap, group)
+		cache.updatedAt = time.Now()
 		return nil
 	}
 
@@ -749,7 +773,10 @@ func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enab
 	if err := store.saveGroupInfo(ctx, group, gi); err != nil {
 		return err
 	}
+
 	cache.iamGroupsMap[group] = gi
+	cache.updatedAt = time.Now()
+
 	return nil
 }
 
@@ -800,7 +827,7 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 			return
 		}
 		cache.iamGroupsMap = m
-
+		cache.updatedAt = time.Now()
 		for k := range cache.iamGroupsMap {
 			res = append(res, k)
 		}
@@ -813,6 +840,7 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 			return
 		}
 		cache.iamGroupPolicyMap = m
+		cache.updatedAt = time.Now()
 		for k := range cache.iamGroupPolicyMap {
 			res = append(res, k)
 		}
@@ -861,6 +889,8 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		} else {
 			delete(cache.iamGroupPolicyMap, name)
 		}
+		cache.updatedAt = time.Now()
+
 		return nil
 	}
 
@@ -881,6 +911,7 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 	} else {
 		cache.iamGroupPolicyMap[name] = mp
 	}
+	cache.updatedAt = time.Now()
 	return nil
 }
 
@@ -930,6 +961,7 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
 			cache.iamGroupPolicyMap[g] = newMappedPolicy(strings.Join(pset.ToSlice(), ","))
 		}
 
+		cache.updatedAt = time.Now()
 		return nil
 	}
 	return err
@@ -987,6 +1019,8 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error
 	}
 
 	delete(cache.iamPolicyDocsMap, policy)
+	cache.updatedAt = time.Now()
+
 	return nil
 }
 
@@ -1057,6 +1091,8 @@ func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iam
 	}
 
 	cache.iamPolicyDocsMap[name] = d
+	cache.updatedAt = time.Now()
+
 	return nil
 }
 
@@ -1076,6 +1112,7 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 	setDefaultCannedPolicies(m)
 
 	cache.iamPolicyDocsMap = m
+	cache.updatedAt = time.Now()
 
 	ret := map[string]iampolicy.Policy{}
 	for k, v := range m {
@@ -1103,6 +1140,7 @@ func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string)
 	setDefaultCannedPolicies(m)
 
 	cache.iamPolicyDocsMap = m
+	cache.updatedAt = time.Now()
 
 	ret := map[string]PolicyDoc{}
 	for k, v := range m {
@@ -1299,6 +1337,8 @@ func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context,
 		// This means that the policy mapping was deleted, so we update
 		// the cache.
 		delete(m, userOrGroup)
+		cache.updatedAt = time.Now()
+
 		err = nil
 	}
 	return err
@@ -1347,11 +1387,15 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 
 		// 3. Delete any mapped policy
 		delete(cache.iamUserPolicyMap, accessKey)
+
+		cache.updatedAt = time.Now()
 		return nil
 	}
+
 	if err != nil {
 		return err
 	}
+
 	if userType != svcUser {
 		err = store.loadMappedPolicy(ctx, accessKey, userType, false, cache.iamUserPolicyMap)
 		// Ignore policy not mapped error
@@ -1373,6 +1417,7 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		if cred.IsTemp() && cred.ParentUser != "" && cred.ParentUser != globalActiveCred.AccessKey {
 			if _, ok := cache.iamUserPolicyMap[cred.ParentUser]; !ok {
 				cache.iamUserPolicyMap[cred.ParentUser] = cache.iamUserPolicyMap[accessKey]
+				cache.updatedAt = time.Now()
 			}
 		}
 	}
@@ -1430,6 +1475,8 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	}
 	delete(cache.iamUsersMap, accessKey)
 
+	cache.updatedAt = time.Now()
+
 	return err
 }
 
@@ -1469,6 +1516,9 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 	}
 
 	cache.iamUsersMap[accessKey] = cred
+
+	cache.updatedAt = time.Now()
+
 	return nil
 }
 
@@ -1479,6 +1529,7 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 	cache := store.lock()
 	defer store.unlock()
 
+	var deleted bool
 	usersToDelete := set.CreateStringSet(users...)
 	for user, cred := range cache.iamUsersMap {
 		userType := regUser
@@ -1497,9 +1548,15 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 			err := store.deleteUserIdentity(ctx, user, userType)
 			logger.LogIf(GlobalContext, err)
 			delete(cache.iamUsersMap, user)
+
+			deleted = true
 		}
 	}
 
+	if deleted {
+		cache.updatedAt = time.Now()
+	}
+
 	return nil
 }
 
@@ -1610,6 +1667,8 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
 	}
 
 	cache.iamUsersMap[accessKey] = uinfo.Credentials
+	cache.updatedAt = time.Now()
+
 	return nil
 }
 
@@ -1643,6 +1702,7 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 	}
 
 	cache.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+	cache.updatedAt = time.Now()
 
 	return nil
 }
@@ -1725,6 +1785,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	}
 
 	cache.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+	cache.updatedAt = time.Now()
 
 	return nil
 }
@@ -1734,14 +1795,29 @@ func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string
 	cache := store.rlock()
 	defer store.runlock()
 
+	userExists := false
 	var tempAccounts []auth.Credentials
 	for _, v := range cache.iamUsersMap {
-		if v.IsTemp() && v.ParentUser == accessKey {
-			// Hide secret key & session key here
-			v.SecretKey = ""
-			v.SessionToken = ""
-			tempAccounts = append(tempAccounts, v)
+		isDerived := false
+		if v.IsServiceAccount() || v.IsTemp() {
+			isDerived = true
 		}
+
+		if !isDerived && v.AccessKey == accessKey {
+			userExists = true
+		} else if isDerived && v.ParentUser == accessKey {
+			userExists = true
+			if v.IsTemp() {
+				// Hide secret key & session key here
+				v.SecretKey = ""
+				v.SessionToken = ""
+				tempAccounts = append(tempAccounts, v)
+			}
+		}
+	}
+
+	if !userExists {
+		return nil, errNoSuchUser
 	}
 
 	return tempAccounts, nil
@@ -1752,14 +1828,29 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 	cache := store.rlock()
 	defer store.runlock()
 
+	userExists := false
 	var serviceAccounts []auth.Credentials
 	for _, v := range cache.iamUsersMap {
-		if v.IsServiceAccount() && v.ParentUser == accessKey {
-			// Hide secret key & session key here
-			v.SecretKey = ""
-			v.SessionToken = ""
-			serviceAccounts = append(serviceAccounts, v)
+		isDerived := false
+		if v.IsServiceAccount() || v.IsTemp() {
+			isDerived = true
 		}
+
+		if !isDerived && v.AccessKey == accessKey {
+			userExists = true
+		} else if isDerived && v.ParentUser == accessKey {
+			userExists = true
+			if v.IsServiceAccount() {
+				// Hide secret key & session key here
+				v.SecretKey = ""
+				v.SessionToken = ""
+				serviceAccounts = append(serviceAccounts, v)
+			}
+		}
+	}
+
+	if !userExists {
+		return nil, errNoSuchUser
 	}
 
 	return serviceAccounts, nil
@@ -1770,6 +1861,8 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
 	cache := store.lock()
 	defer store.unlock()
 
+	cache.updatedAt = time.Now()
+
 	cr, ok := cache.iamUsersMap[accessKey]
 
 	// It is not possible to update an STS account.
@@ -1803,6 +1896,8 @@ func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, se
 	cache := store.lock()
 	defer store.unlock()
 
+	cache.updatedAt = time.Now()
+
 	cred, ok := cache.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
@@ -1837,6 +1932,8 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 	cache := store.lock()
 	defer store.unlock()
 
+	cache.updatedAt = time.Now()
+
 	userType := regUser
 	if cred.IsServiceAccount() {
 		userType = svcUser
@@ -1858,6 +1955,8 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 	cache := store.lock()
 	defer store.unlock()
 
+	cache.updatedAt = time.Now()
+
 	_, found := cache.iamUsersMap[accessKey]
 	if !found {
 		store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)

cmd/iam.go

@@ -38,6 +38,8 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	xldap "github.com/minio/minio/internal/config/identity/ldap"
+	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -80,6 +82,8 @@ type IAMSys struct {
 	sync.Mutex
 
 	iamRefreshInterval time.Duration
+	ldapConfig         xldap.Config  // only valid if usersSysType is LDAPUsers
+	openIDConfig       openid.Config // only valid if OpenID is configured
 
 	usersSysType UsersSysType
 
@@ -164,7 +168,7 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 
 // initStore initializes IAM stores
 func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
-	if globalLDAPConfig.Enabled {
+	if sys.ldapConfig.Enabled {
 		sys.EnableLDAPSys()
 	}
 
@@ -224,6 +228,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	sys.Lock()
 	defer sys.Unlock()
 
+	sys.ldapConfig = globalLDAPConfig.Clone()
+	sys.openIDConfig = globalOpenIDConfig.Clone()
 	sys.iamRefreshInterval = iamRefreshInterval
 
 	// Initialize IAM store
@@ -310,28 +316,33 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	// Set up polling for expired accounts and credentials purging.
 	switch {
-	case globalOpenIDConfig.ProviderEnabled():
+	case sys.openIDConfig.ProviderEnabled():
 		go func() {
-			ticker := time.NewTicker(sys.iamRefreshInterval)
-			defer ticker.Stop()
+			timer := time.NewTimer(sys.iamRefreshInterval)
+			defer timer.Stop()
 			for {
 				select {
-				case <-ticker.C:
+				case <-timer.C:
 					sys.purgeExpiredCredentialsForExternalSSO(ctx)
+
+					timer.Reset(sys.iamRefreshInterval)
 				case <-ctx.Done():
 					return
 				}
 			}
 		}()
-	case globalLDAPConfig.Enabled:
+	case sys.ldapConfig.Enabled:
 		go func() {
-			ticker := time.NewTicker(sys.iamRefreshInterval)
-			defer ticker.Stop()
+			timer := time.NewTimer(sys.iamRefreshInterval)
+			defer timer.Stop()
+
 			for {
 				select {
-				case <-ticker.C:
+				case <-timer.C:
 					sys.purgeExpiredCredentialsForLDAP(ctx)
 					sys.updateGroupMembershipsForLDAP(ctx)
+
+					timer.Reset(sys.iamRefreshInterval)
 				case <-ctx.Done():
 					return
 				}
@@ -343,7 +354,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	go sys.watch(ctx)
 
 	// Load RoleARN
-	if rolePolicyMap := globalOpenIDConfig.GetRoleInfo(); rolePolicyMap != nil {
+	if rolePolicyMap := sys.openIDConfig.GetRoleInfo(); rolePolicyMap != nil {
 		// Validate that policies associated with roles are defined.
 		for _, rolePolicies := range rolePolicyMap {
 			ps := newMappedPolicy(rolePolicies).toSlice()
@@ -403,12 +414,12 @@ func (sys *IAMSys) watch(ctx context.Context) {
 
 	var maxRefreshDurationSecondsForLog float64 = 10
 
-	// Fall back to loading all items periodically
-	ticker := time.NewTicker(sys.iamRefreshInterval)
-	defer ticker.Stop()
+	// Load all items periodically
+	timer := time.NewTimer(sys.iamRefreshInterval)
+	defer timer.Stop()
 	for {
 		select {
-		case <-ticker.C:
+		case <-timer.C:
 			refreshStart := time.Now()
 			if err := sys.Load(ctx); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
@@ -420,6 +431,7 @@ func (sys *IAMSys) watch(ctx context.Context) {
 				}
 			}
 
+			timer.Reset(sys.iamRefreshInterval)
 		case <-ctx.Done():
 			return
 		}
@@ -681,7 +693,7 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 		return errServerNotInitialized
 	}
 
-	if globalPolicyOPA != nil {
+	if globalAuthZPlugin != nil {
 		// If OPA is set, we do not need to set a policy mapping.
 		policyName = ""
 	}
@@ -1132,7 +1144,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 			continue
 		}
 		roleArn = roleArns[0]
-		u, err := globalOpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
+		u, err := sys.openIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue
@@ -1154,14 +1166,14 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var allDistNames []string
 	for parentUser := range parentUsers {
-		if !globalLDAPConfig.IsLDAPUserDN(parentUser) {
+		if !sys.ldapConfig.IsLDAPUserDN(parentUser) {
 			continue
 		}
 
 		allDistNames = append(allDistNames, parentUser)
 	}
 
-	expiredUsers, err := globalLDAPConfig.GetNonEligibleUserDistNames(allDistNames)
+	expiredUsers, err := sys.ldapConfig.GetNonEligibleUserDistNames(allDistNames)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1183,7 +1195,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// DN to ldap username mapping for each LDAP user
 	parentUserToLDAPUsernameMap := make(map[string]string)
 	for _, cred := range allCreds {
-		if !globalLDAPConfig.IsLDAPUserDN(cred.ParentUser) {
+		if !sys.ldapConfig.IsLDAPUserDN(cred.ParentUser) {
 			continue
 		}
 		// Check if this is the first time we are
@@ -1231,7 +1243,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	}
 
 	// 2. Query LDAP server for groups of the LDAP users collected.
-	updatedGroups, err := globalLDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	updatedGroups, err := sys.ldapConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1537,54 +1549,10 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 	return combinedPolicy.IsAllowed(parentArgs) && subPolicy.IsAllowed(parentArgs)
 }
 
-// IsAllowedLDAPSTS - checks for LDAP specific claims and values
-func (sys *IAMSys) IsAllowedLDAPSTS(args iampolicy.Args, parentUser string) bool {
-	// parentUser value must match the ldap user in the claim.
-	if parentInClaimIface, ok := args.Claims[ldapUser]; !ok {
-		// no ldapUser claim present reject it.
-		return false
-	} else if parentInClaim, ok := parentInClaimIface.(string); !ok {
-		// not the right type, reject it.
-		return false
-	} else if parentInClaim != parentUser {
-		// ldap claim has been modified maliciously reject it.
-		return false
-	}
-
-	// Check policy for this LDAP user.
-	ldapPolicies, err := sys.PolicyDBGet(parentUser, false, args.Groups...)
-	if err != nil {
-		return false
-	}
-
-	if len(ldapPolicies) == 0 {
-		return false
-	}
-
-	// Policies were found, evaluate all of them.
-	availablePoliciesStr, combinedPolicy := sys.store.FilterPolicies(strings.Join(ldapPolicies, ","), "")
-	if availablePoliciesStr == "" {
-		return false
-	}
-
-	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args)
-	if hasSessionPolicy {
-		return isAllowedSP && combinedPolicy.IsAllowed(args)
-	}
-
-	return combinedPolicy.IsAllowed(args)
-}
-
 // IsAllowedSTS is meant for STS based temporary credentials,
 // which implements claims validation and verification other than
 // applying policies.
 func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
-	// If it is an LDAP request, check that user and group
-	// policies allow the request.
-	if sys.usersSysType == LDAPUsersSysType {
-		return sys.IsAllowedLDAPSTS(args, parentUser)
-	}
-
 	var policies []string
 	roleArn := args.GetRoleArn()
 	if roleArn != "" {
@@ -1619,6 +1587,11 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 		}
 	}
 
+	// Defensive code: Do not allow any operation if no policy is found in the session token
+	if len(policies) == 0 {
+		return false
+	}
+
 	combinedPolicy, err := sys.store.GetPolicy(strings.Join(policies, ","))
 	if err == errNoSuchPolicy {
 		for _, pname := range policies {
@@ -1693,8 +1666,8 @@ func (sys *IAMSys) GetCombinedPolicy(policies ...string) iampolicy.Policy {
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	// If opa is configured, use OPA always.
-	if globalPolicyOPA != nil {
-		ok, err := globalPolicyOPA.IsAllowed(args)
+	if globalAuthZPlugin != nil {
+		ok, err := globalAuthZPlugin.IsAllowed(args)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 		}

cmd/local-locker_test.go

@@ -344,3 +344,96 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 		})
 	}
 }
+
+func Test_localLocker_RUnlock(t *testing.T) {
+	rng := rand.New(rand.NewSource(0))
+	// Numbers of unique locks
+	for _, locks := range []int{1, 100, 1000, 1e6} {
+		if testing.Short() && locks > 100 {
+			continue
+		}
+		t.Run(fmt.Sprintf("%d-locks", locks), func(t *testing.T) {
+			// Number of readers per lock...
+			for _, readers := range []int{1, 10, 100} {
+				if locks > 1000 && readers > 1 {
+					continue
+				}
+				if testing.Short() && readers > 10 {
+					continue
+				}
+				t.Run(fmt.Sprintf("%d-read", readers), func(t *testing.T) {
+					l := newLocker()
+					for i := 0; i < locks; i++ {
+						var tmp [16]byte
+						rng.Read(tmp[:])
+						res := []string{hex.EncodeToString(tmp[:])}
+
+						for i := 0; i < readers; i++ {
+							rng.Read(tmp[:])
+							ok, err := l.RLock(context.Background(), dsync.LockArgs{
+								UID:       uuid.NewString(),
+								Resources: res,
+								Source:    hex.EncodeToString(tmp[:8]),
+								Owner:     hex.EncodeToString(tmp[8:]),
+								Quorum:    0,
+							})
+							if !ok || err != nil {
+								t.Fatal("failed:", err, ok)
+							}
+						}
+					}
+
+					// Expire 50%
+					toUnLock := make([]dsync.LockArgs, 0, locks*readers)
+					for k, v := range l.lockMap {
+						for _, lock := range v {
+							if rng.Intn(2) == 0 {
+								toUnLock = append(toUnLock, dsync.LockArgs{Resources: []string{k}, UID: lock.UID})
+							}
+						}
+					}
+					start := time.Now()
+					for _, lock := range toUnLock {
+						ok, err := l.ForceUnlock(context.Background(), lock)
+						if err != nil || !ok {
+							t.Fatal(err)
+						}
+					}
+					t.Logf("Expire 50%% took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+
+					if len(l.lockUID) == locks*readers {
+						t.Fatalf("objects uids all remain, unlikely")
+					}
+					if len(l.lockMap) == 0 && locks > 10 {
+						t.Fatalf("objects all deleted, 0 remains")
+					}
+					if len(l.lockUID) != locks*readers-len(toUnLock) {
+						t.Fatalf("want %d objects uids all deleted, %d remains", len(l.lockUID), locks*readers-len(toUnLock))
+					}
+
+					toUnLock = toUnLock[:0]
+					for k, v := range l.lockMap {
+						for _, lock := range v {
+							toUnLock = append(toUnLock, dsync.LockArgs{Resources: []string{k}, UID: lock.UID, Owner: lock.Owner})
+						}
+					}
+					start = time.Now()
+					for _, lock := range toUnLock {
+						ok, err := l.RUnlock(nil, lock)
+						if err != nil || !ok {
+							t.Fatal(err)
+						}
+					}
+					t.Logf("Expire rest took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+
+					if len(l.lockMap) != 0 {
+						t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockMap))
+					}
+					if len(l.lockUID) != 0 {
+						t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockUID))
+					}
+				})
+			}
+		})
+	}
+}

cmd/lock-rest-server.go

@@ -239,10 +239,10 @@ func lockMaintenance(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-lkTimer.C:
+			globalLockServer.expireOldLocks(lockValidityDuration)
+
 			// Reset the timer for next cycle.
 			lkTimer.Reset(lockMaintenanceInterval)
-
-			globalLockServer.expireOldLocks(lockValidityDuration)
 		}
 	}
 }

cmd/metacache-set.go

@@ -882,7 +882,11 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			}
 
 			// fallback only when set.
-			for fd := fallback(werr); fd != nil; {
+			for {
+				fd := fallback(werr)
+				if fd == nil {
+					break
+				}
 				// This fallback is only set when
 				// askDisks is less than total
 				// number of disks per set.

cmd/metrics-v2.go

@@ -1604,8 +1604,9 @@ func getBucketUsageMetrics() *MetricsGroup {
 			Value:       float64(time.Since(dataUsageInfo.LastUpdate)),
 		})
 
+		bucketReplStats := getAllLatestReplicationStats(dataUsageInfo.BucketsUsage)
 		for bucket, usage := range dataUsageInfo.BucketsUsage {
-			stats := getLatestReplicationStats(bucket, usage)
+			stats := bucketReplStats[bucket]
 
 			quota, _ := globalBucketQuotaSys.Get(ctx, bucket)
 

cmd/mrf.go

@@ -190,12 +190,12 @@ func (m *mrfState) healRoutine() {
 	}
 
 	for {
-		idler.Reset(mrfInfoResetInterval)
 		select {
 		case <-m.ctx.Done():
 			return
 		case <-idler.C:
 			m.resetMRFInfoIfNoPendingOps()
+			idler.Reset(mrfInfoResetInterval)
 		case setInfo := <-m.setReconnectEvent:
 			// Get the list of objects related the er.set
 			// to which the connected disk belongs.
@@ -218,21 +218,19 @@ func (m *mrfState) healRoutine() {
 
 			// Heal objects
 			for _, u := range mrfOperations {
-				if _, err := m.objectAPI.HealObject(m.ctx, u.bucket, u.object, u.versionID, mrfHealingOpts); err != nil {
-					// If not deleted, assume they failed.
-					logger.LogIf(m.ctx, err)
-				} else {
-					m.mu.Lock()
-					m.itemsHealed++
-					m.pendingItems--
-					m.bytesHealed += uint64(u.size)
-					m.pendingBytes -= uint64(u.size)
-					m.mu.Unlock()
-				}
-
+				_, err := m.objectAPI.HealObject(m.ctx, u.bucket, u.object, u.versionID, mrfHealingOpts)
 				m.mu.Lock()
+				if err == nil {
+					m.itemsHealed++
+					m.bytesHealed += uint64(u.size)
+				}
+				m.pendingItems--
+				m.pendingBytes -= uint64(u.size)
 				delete(m.pendingOps, u)
 				m.mu.Unlock()
+
+				// Log healing error if any
+				logger.LogIf(m.ctx, err)
 			}
 
 			waitForLowHTTPReq()

cmd/notification.go

@@ -414,6 +414,21 @@ func (sys *NotificationSys) ServerUpdate(ctx context.Context, u *url.URL, sha256
 	return ng.Wait()
 }
 
+// SignalConfigReload reloads requested sub-system on a remote peer dynamically.
+func (sys *NotificationSys) SignalConfigReload(subSys string) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(GlobalContext, func() error {
+			return client.SignalService(serviceReloadDynamic, subSys)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
 // SignalService - calls signal service RPC call on all peers.
 func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
@@ -423,7 +438,7 @@ func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerE
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(sig)
+			return client.SignalService(sig, "")
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -586,6 +601,44 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
 	}
 }
 
+// GetClusterAllBucketStats - returns bucket stats for all buckets from all remote peers.
+func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []BucketStatsMap {
+	ng := WithNPeers(len(sys.peerClients))
+	replicationStats := make([]BucketStatsMap, len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		index := index
+		client := client
+		ng.Go(ctx, func() error {
+			if client == nil {
+				return errPeerNotReachable
+			}
+			bsMap, err := client.GetAllBucketStats()
+			if err != nil {
+				return err
+			}
+			replicationStats[index] = bsMap
+			return nil
+		}, index, *client.host)
+	}
+	for _, nErr := range ng.Wait() {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
+		if nErr.Err != nil {
+			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+		}
+	}
+
+	replicationStatsList := globalReplicationStats.GetAll()
+	bucketStatsMap := make(map[string]BucketStats, len(replicationStatsList))
+	for k, replicationStats := range replicationStatsList {
+		bucketStatsMap[k] = BucketStats{
+			ReplicationStats: replicationStats,
+		}
+	}
+
+	replicationStats = append(replicationStats, BucketStatsMap(bucketStatsMap))
+	return replicationStats
+}
+
 // GetClusterBucketStats - calls GetClusterBucketStats call on all peers for a cluster statistics view.
 func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketName string) []BucketStats {
 	ng := WithNPeers(len(sys.peerClients))
@@ -1541,7 +1594,7 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(serviceSig)
+			return client.SignalService(serviceSig, "")
 		}, idx, *client.host)
 	}
 	nerrs := ng.Wait()

cmd/object-api-interface.go

@@ -76,6 +76,8 @@ type ObjectOptions struct {
 	// Mutate set to 'true' if the call is namespace mutation call
 	Mutate        bool
 	WalkAscending bool // return Walk results in ascending order of versions
+
+	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/object-api-listobjects_test.go

@@ -70,7 +70,7 @@ func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 Te
 		md5Bytes := md5.Sum([]byte(object.content))
 		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{
-			Versioned:   globalBucketVersioningSys.Enabled(object.parentBucket),
+			Versioned:   globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name),
 			UserDefined: object.meta,
 		})
 		if err != nil {
@@ -78,7 +78,7 @@ func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 Te
 		}
 		if object.addDeleteMarker {
 			oi, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{
-				Versioned: globalBucketVersioningSys.Enabled(object.parentBucket),
+				Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name),
 			})
 			if err != nil {
 				t.Fatalf("%s : %s", instanceType, err.Error())
@@ -380,7 +380,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name,
 			mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{
-				Versioned:   globalBucketVersioningSys.Enabled(object.parentBucket),
+				Versioned:   globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name),
 				UserDefined: object.meta,
 			})
 		if err != nil {
@@ -1065,16 +1065,16 @@ func testDeleteObjectVersion(obj ObjectLayer, instanceType string, t1 TestErrHan
 		_, err := obj.PutObject(context.Background(), object.parentBucket, object.name,
 			mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{
-				Versioned:        globalBucketVersioningSys.Enabled(object.parentBucket),
-				VersionSuspended: globalBucketVersioningSys.Suspended(object.parentBucket),
+				Versioned:        globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name),
+				VersionSuspended: globalBucketVersioningSys.PrefixSuspended(object.parentBucket, object.name),
 				UserDefined:      object.meta,
 			})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}
 		obj, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{
-			Versioned:        globalBucketVersioningSys.Enabled(object.parentBucket),
-			VersionSuspended: globalBucketVersioningSys.Suspended(object.parentBucket),
+			Versioned:        globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name),
+			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(object.parentBucket, object.name),
 			VersionID:        object.versionID,
 		})
 		if err != nil {

cmd/object-api-options.go

@@ -157,16 +157,16 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 			}
 		}
 	}
+	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(bucket, object)
 	return opts, nil
 }
 
 func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
-	versioned := globalBucketVersioningSys.Enabled(bucket)
 	opts, err = getOpts(ctx, r, bucket, object)
 	if err != nil {
 		return opts, err
 	}
-	opts.Versioned = versioned
+	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket)
 	delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
 	if delMarker != "" {
@@ -203,8 +203,9 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
 
 // get ObjectOptions for PUT calls from encryption headers and metadata
 func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
+	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
+
 	vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)

cmd/object-handlers-common.go

@@ -266,8 +266,6 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
 }
 
 func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete) {
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
 	for remaining := toDel; len(remaining) > 0; toDel = remaining {
 		if len(toDel) > maxDeleteList {
 			remaining = toDel[maxDeleteList:]
@@ -275,9 +273,10 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 		} else {
 			remaining = nil
 		}
+		vc, _ := globalBucketVersioningSys.Get(bucket)
 		deletedObjs, errs := o.DeleteObjects(ctx, bucket, toDel, ObjectOptions{
-			Versioned:        versioned,
-			VersionSuspended: versionSuspended,
+			PrefixEnabledFn:  vc.PrefixEnabled,
+			VersionSuspended: vc.Suspended(),
 		})
 		var logged bool
 		for i, err := range errs {

cmd/object-handlers.go

@@ -204,7 +204,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
-		if globalBucketVersioningSys.Enabled(bucket) {
+		if globalBucketVersioningSys.PrefixEnabled(bucket, object) {
 			// Versioning enabled quite possibly object is deleted might be delete-marker
 			// if present set the headers, no idea why AWS S3 sets these headers.
 			if objInfo.VersionID != "" && objInfo.DeleteMarker {
@@ -425,7 +425,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			proxy  proxyResult
 			perr   error
 		)
-		proxytgts := getproxyTargets(ctx, bucket, object, opts)
+		proxytgts := getProxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
 			// proxy to replication target if active-active replication is in place.
 			reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
@@ -445,7 +445,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 				writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL)
 				return
 			}
-			if globalBucketVersioningSys.Enabled(bucket) && gr != nil {
+			if globalBucketVersioningSys.PrefixEnabled(bucket, object) && gr != nil {
 				if !gr.ObjInfo.VersionPurgeStatus.Empty() {
 					// Shows the replication status of a permanent delete of a version
 					w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(gr.ObjInfo.VersionPurgeStatus)}
@@ -662,7 +662,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 			oi    ObjectInfo
 		)
 		// proxy HEAD to replication target if active-active replication configured on bucket
-		proxytgts := getproxyTargets(ctx, bucket, object, opts)
+		proxytgts := getProxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
 			if rangeHeader != "" {
 				rs, _ = parseRequestRangeSpec(rangeHeader)
@@ -673,7 +673,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 			}
 		}
 		if !proxy.Proxy {
-			if globalBucketVersioningSys.Enabled(bucket) {
+			if globalBucketVersioningSys.PrefixEnabled(bucket, object) {
 				switch {
 				case !objInfo.VersionPurgeStatus.Empty():
 					w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(objInfo.VersionPurgeStatus)}
@@ -1087,7 +1087,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if isErrPreconditionFailed(err) {
 			return
 		}
-		if globalBucketVersioningSys.Enabled(srcBucket) && gr != nil {
+		if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil {
 			// Versioning enabled quite possibly object is deleted might be delete-marker
 			// if present set the headers, no idea why AWS S3 sets these headers.
 			if gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker {
@@ -1424,6 +1424,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		srcInfo.metadataOnly = false
 	}
 
+	if srcInfo.metadataOnly {
+		gr.Close() // We are not interested in the reader stream at this point close it.
+	}
+
 	// Check if x-amz-metadata-directive or x-amz-tagging-directive was not set to REPLACE and source,
 	// destination are same objects. Apply this restriction also when
 	// metadataOnly is true indicating that we are not overwriting the object.
@@ -2480,7 +2484,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		if isErrPreconditionFailed(err) {
 			return
 		}
-		if globalBucketVersioningSys.Enabled(srcBucket) && gr != nil {
+		if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil {
 			// Versioning enabled quite possibly object is deleted might be delete-marker
 			// if present set the headers, no idea why AWS S3 sets these headers.
 			if gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker {
@@ -3211,8 +3215,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		w.Write(encodedErrorResponse)
 	}
 
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	suspended := globalBucketVersioningSys.Suspended(bucket)
+	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
+	suspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 	os := newObjSweeper(bucket, object).WithVersioning(versioned, suspended)
 	if !globalTierConfigMgr.Empty() {
 		// Get appropriate object info to identify the remote object to delete

cmd/peer-rest-client.go

@@ -510,6 +510,19 @@ func (client *peerRESTClient) GetBucketStats(bucket string) (BucketStats, error)
 	return bs, msgp.Decode(respBody, &bs)
 }
 
+// GetAllBucketStats - load replication stats for all buckets
+func (client *peerRESTClient) GetAllBucketStats() (BucketStatsMap, error) {
+	values := make(url.Values)
+	respBody, err := client.call(peerRESTMethodGetAllBucketStats, values, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+
+	bsMap := BucketStatsMap{}
+	defer http.DrainBody(respBody)
+	return bsMap, msgp.Decode(respBody, &bsMap)
+}
+
 // LoadBucketMetadata - load bucket metadata
 func (client *peerRESTClient) LoadBucketMetadata(bucket string) error {
 	values := make(url.Values)
@@ -688,9 +701,10 @@ func (client *peerRESTClient) ServerUpdate(ctx context.Context, u *url.URL, sha2
 }
 
 // SignalService - sends signal to peer nodes.
-func (client *peerRESTClient) SignalService(sig serviceSignal) error {
+func (client *peerRESTClient) SignalService(sig serviceSignal, subSys string) error {
 	values := make(url.Values)
 	values.Set(peerRESTSignal, strconv.Itoa(int(sig)))
+	values.Set(peerRESTSubSys, subSys)
 	respBody, err := client.call(peerRESTMethodSignalService, values, nil, -1)
 	if err != nil {
 		return err
@@ -903,43 +917,54 @@ func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh <-chan stru
 	}()
 }
 
+func (client *peerRESTClient) doConsoleLog(logCh chan interface{}, doneCh <-chan struct{}) {
+	// To cancel the REST request in case doneCh gets closed.
+	ctx, cancel := context.WithCancel(GlobalContext)
+
+	cancelCh := make(chan struct{})
+	defer close(cancelCh)
+	go func() {
+		select {
+		case <-doneCh:
+		case <-cancelCh:
+			// There was an error in the REST request.
+		}
+		cancel()
+	}()
+
+	respBody, err := client.callWithContext(ctx, peerRESTMethodLog, nil, nil, -1)
+	defer http.DrainBody(respBody)
+	if err != nil {
+		return
+	}
+
+	dec := gob.NewDecoder(respBody)
+	for {
+		var lg madmin.LogInfo
+		if err = dec.Decode(&lg); err != nil {
+			break
+		}
+		if lg.DeploymentID != "" {
+			select {
+			case logCh <- lg:
+			default:
+				// Do not block on slow receivers.
+			}
+		}
+	}
+}
+
 // ConsoleLog - sends request to peer nodes to get console logs
 func (client *peerRESTClient) ConsoleLog(logCh chan interface{}, doneCh <-chan struct{}) {
 	go func() {
 		for {
-			// get cancellation context to properly unsubscribe peers
-			ctx, cancel := context.WithCancel(GlobalContext)
-			respBody, err := client.callWithContext(ctx, peerRESTMethodLog, nil, nil, -1)
-			if err != nil {
-				// Retry the failed request.
-				time.Sleep(5 * time.Second)
-			} else {
-				dec := gob.NewDecoder(respBody)
-
-				go func() {
-					<-doneCh
-					cancel()
-				}()
-
-				for {
-					var log madmin.LogInfo
-					if err = dec.Decode(&log); err != nil {
-						break
-					}
-					select {
-					case logCh <- log:
-					default:
-					}
-				}
-			}
-
+			client.doConsoleLog(logCh, doneCh)
 			select {
 			case <-doneCh:
-				cancel()
-				http.DrainBody(respBody)
 				return
 			default:
-				// There was error in the REST request, retry.
+				// There was error in the REST request, retry after sometime as probably the peer is down.
+				time.Sleep(5 * time.Second)
 			}
 		}
 	}()

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v21" // Add netperf
+	peerRESTVersion       = "v22" // Add bulk GetBucketStats
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -41,6 +41,7 @@ const (
 	peerRESTMethodDeleteBucketMetadata        = "/deletebucketmetadata"
 	peerRESTMethodLoadBucketMetadata          = "/loadbucketmetadata"
 	peerRESTMethodGetBucketStats              = "/getbucketstats"
+	peerRESTMethodGetAllBucketStats           = "/getallbucketstats"
 	peerRESTMethodServerUpdate                = "/serverupdate"
 	peerRESTMethodSignalService               = "/signalservice"
 	peerRESTMethodBackgroundHealStatus        = "/backgroundhealstatus"
@@ -84,6 +85,7 @@ const (
 	peerRESTUserOrGroup    = "user-or-group"
 	peerRESTIsGroup        = "is-group"
 	peerRESTSignal         = "signal"
+	peerRESTSubSys         = "sub-sys"
 	peerRESTProfiler       = "profiler"
 	peerRESTTraceErr       = "err"
 	peerRESTTraceInternal  = "internal"

cmd/peer-rest-server.go

@@ -561,8 +561,26 @@ func (s *peerRESTServer) ReloadSiteReplicationConfigHandler(w http.ResponseWrite
 	logger.LogIf(r.Context(), globalSiteReplicationSys.Init(ctx, objAPI))
 }
 
+// GetAllBucketStatsHandler - fetches bucket replication stats for all buckets from this peer.
+func (s *peerRESTServer) GetAllBucketStatsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	replicationStats := globalReplicationStats.GetAll()
+	bucketStatsMap := make(map[string]BucketStats, len(replicationStats))
+	for k, v := range replicationStats {
+		bucketStatsMap[k] = BucketStats{
+			ReplicationStats: v,
+		}
+	}
+
+	logger.LogIf(r.Context(), msgp.Encode(w, BucketStatsMap(bucketStatsMap)))
+}
+
 // GetBucketStatsHandler - fetches current in-memory bucket stats, currently only
-// returns BucketReplicationStatus
+// returns BucketStats, that currently includes ReplicationStats.
 func (s *peerRESTServer) GetBucketStatsHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
@@ -837,7 +855,14 @@ func (s *peerRESTServer) SignalServiceHandler(w http.ResponseWriter, r *http.Req
 			s.writeErrorResponse(w, err)
 			return
 		}
-		if err = applyDynamicConfig(r.Context(), objAPI, srvCfg); err != nil {
+		subSys := r.Form.Get(peerRESTSubSys)
+		// Apply dynamic values.
+		if subSys == "" {
+			err = applyDynamicConfig(r.Context(), objAPI, srvCfg)
+		} else {
+			err = applyDynamicConfigForSubSys(r.Context(), objAPI, srvCfg, subSys)
+		}
+		if err != nil {
 			s.writeErrorResponse(w, err)
 		}
 		return
@@ -932,6 +957,8 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			w.(http.Flusher).Flush()
+		case <-r.Context().Done():
+			return
 		case <-keepAliveTicker.C:
 			if err := enc.Encode(&event.Event{}); err != nil {
 				return
@@ -993,6 +1020,8 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			w.(http.Flusher).Flush()
+		case <-r.Context().Done():
+			return
 		case <-keepAliveTicker.C:
 			if err := enc.Encode(&madmin.TraceInfo{}); err != nil {
 				return
@@ -1059,15 +1088,15 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	w.Header().Set("Connection", "close")
-	w.WriteHeader(http.StatusOK)
-
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
 	ch := make(chan interface{}, 2000)
 	globalConsoleSys.Subscribe(ch, doneCh, "", 0, string(logger.All), nil)
 
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
 	enc := gob.NewEncoder(w)
 	for {
 		select {
@@ -1076,6 +1105,11 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 				return
 			}
 			w.(http.Flusher).Flush()
+		case <-keepAliveTicker.C:
+			if err := enc.Encode(&madmin.LogInfo{}); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
 		case <-r.Context().Done():
 			return
 		}
@@ -1300,6 +1334,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetInfo).HandlerFunc(httpTraceHdrs(server.NetInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDispatchNetInfo).HandlerFunc(httpTraceHdrs(server.DispatchNetInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCycleBloom).HandlerFunc(httpTraceHdrs(server.CycleServerBloomFilterHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBucketStats).HandlerFunc(httpTraceHdrs(server.GetBucketStatsHandler)).Queries(restQueries(peerRESTBucket)...)

cmd/prepare-storage.go

@@ -70,7 +70,7 @@ var printEndpointError = func() func(Endpoint, error, bool) {
 }()
 
 // Cleans up tmp directory of the local disk.
-func formatErasureCleanupTmp(diskPath string) {
+func bgFormatErasureCleanupTmp(diskPath string) {
 	// Need to move temporary objects left behind from previous run of minio
 	// server to a unique directory under `minioMetaTmpBucket-old` to clean
 	// up `minioMetaTmpBucket` for the current run.
@@ -98,9 +98,8 @@ func formatErasureCleanupTmp(diskPath string) {
 	}
 
 	go removeAll(tmpOld)
-
 	// Renames and schedules for purging all bucket metacache.
-	renameAllBucketMetacache(diskPath)
+	go renameAllBucketMetacache(diskPath)
 }
 
 // Following error message is added to fix a regression in release

cmd/site-replication.go

@@ -126,6 +126,13 @@ func errSRIAMError(err error) SRError {
 	}
 }
 
+func errSRConfigMissingError(err error) SRError {
+	return SRError{
+		Cause: err,
+		Code:  ErrSiteReplicationConfigMissing,
+	}
+}
+
 var errSRObjectLayerNotReady = SRError{
 	Cause: fmt.Errorf("object layer not ready"),
 	Code:  ErrServerNotInitialized,
@@ -1209,6 +1216,23 @@ func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRB
 	return cErr.summaryErr
 }
 
+// PeerBucketVersioningHandler - updates versioning config to local cluster.
+func (c *SiteReplicationSys) PeerBucketVersioningHandler(ctx context.Context, bucket string, versioning *string) error {
+	if versioning != nil {
+		configData, err := base64.StdEncoding.DecodeString(*versioning)
+		if err != nil {
+			return wrapSRErr(err)
+		}
+		err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+		if err != nil {
+			return wrapSRErr(err)
+		}
+		return nil
+	}
+
+	return nil
+}
+
 // PeerBucketPolicyHandler - copies/deletes policy to local cluster.
 func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket string, policy *policy.Policy) error {
 	if policy != nil {
@@ -1848,6 +1872,8 @@ func (c *SiteReplicationSys) isEnabled() bool {
 	return c.enabled
 }
 
+var errMissingSRConfig = fmt.Errorf("Site not found in site replication configuration")
+
 // RemovePeerCluster - removes one or more clusters from site replication configuration.
 func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI ObjectLayer, rreq madmin.SRRemoveReq) (st madmin.ReplicateRemoveStatus, err error) {
 	if !c.isEnabled() {
@@ -1872,7 +1898,7 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 	for _, s := range siteNames {
 		info, ok := peerMap[s]
 		if !ok {
-			return st, errSRInvalidRequest(fmt.Errorf("Site %s not found in site replication configuration", s))
+			return st, errSRConfigMissingError(errMissingSRConfig)
 		}
 		rmvEndpoints = append(rmvEndpoints, info.Endpoint)
 		delete(updatedPeers, info.DeploymentID)
@@ -1898,6 +1924,9 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 				return
 			}
 			if _, err = admClient.SRPeerRemove(ctx, rreq); err != nil {
+				if errors.As(err, &errMissingSRConfig) {
+					return
+				}
 				errs[pi.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", pi.Name, err))
 				return
 			}
@@ -1937,6 +1966,10 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 // InternalRemoveReq - sends an unlink request to peer cluster to remove one or more sites
 // from the site replication configuration.
 func (c *SiteReplicationSys) InternalRemoveReq(ctx context.Context, objectAPI ObjectLayer, rreq madmin.SRRemoveReq) error {
+	if !c.isEnabled() {
+		return errSRNotEnabled
+	}
+
 	ourName := ""
 	peerMap := make(map[string]madmin.PeerInfo)
 	updatedPeers := make(map[string]madmin.PeerInfo)
@@ -1958,7 +1991,7 @@ func (c *SiteReplicationSys) InternalRemoveReq(ctx context.Context, objectAPI Ob
 	for _, s := range siteNames {
 		info, ok := peerMap[s]
 		if !ok {
-			return fmt.Errorf("Site %s not found in site replication configuration", s)
+			return errMissingSRConfig
 		}
 		if info.DeploymentID == globalDeploymentID {
 			unlinkSelf = true
@@ -2135,6 +2168,7 @@ func (c *SiteReplicationSys) SiteReplicationStatus(ctx context.Context, objAPI O
 	for b, stat := range sinfo.BucketStats {
 		for dID, st := range stat {
 			if st.TagMismatch ||
+				st.VersioningConfigMismatch ||
 				st.OLockConfigMismatch ||
 				st.SSEConfigMismatch ||
 				st.PolicyMismatch ||
@@ -2532,7 +2566,8 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 			replCfgs := make([]*sreplication.Config, numSites)
 			quotaCfgs := make([]*madmin.BucketQuota, numSites)
 			sseCfgSet := set.NewStringSet()
-			var tagCount, olockCfgCount, sseCfgCount int
+			versionCfgSet := set.NewStringSet()
+			var tagCount, olockCfgCount, sseCfgCount, versionCfgCount int
 			for i, s := range slc {
 				if s.ReplicationConfig != nil {
 					cfgBytes, err := base64.StdEncoding.DecodeString(*s.ReplicationConfig)
@@ -2545,6 +2580,16 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 					}
 					replCfgs[i] = cfg
 				}
+				if s.Versioning != nil {
+					configData, err := base64.StdEncoding.DecodeString(*s.Versioning)
+					if err != nil {
+						continue
+					}
+					versionCfgCount++
+					if !versionCfgSet.Contains(string(configData)) {
+						versionCfgSet.Add(string(configData))
+					}
+				}
 				if s.QuotaConfig != nil {
 					cfgBytes, err := base64.StdEncoding.DecodeString(*s.QuotaConfig)
 					if err != nil {
@@ -2574,12 +2619,11 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 					policies[i] = plcy
 				}
 				if s.ObjectLockConfig != nil {
-					olockCfgCount++
 					configData, err := base64.StdEncoding.DecodeString(*s.ObjectLockConfig)
 					if err != nil {
 						continue
 					}
-
+					olockCfgCount++
 					if !olockConfigSet.Contains(string(configData)) {
 						olockConfigSet.Add(string(configData))
 					}
@@ -2589,10 +2633,10 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 					if err != nil {
 						continue
 					}
+					sseCfgCount++
 					if !sseCfgSet.Contains(string(configData)) {
 						sseCfgSet.Add(string(configData))
 					}
-					sseCfgCount++
 				}
 				ss, ok := info.StatsSummary[s.DeploymentID]
 				if !ok {
@@ -2612,6 +2656,9 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 				if sseCfgCount > 0 {
 					ss.TotalSSEConfigCount++
 				}
+				if versionCfgCount > 0 {
+					ss.TotalVersioningConfigCount++
+				}
 				if len(policies) > 0 {
 					ss.TotalBucketPoliciesCount++
 				}
@@ -2620,6 +2667,7 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 			tagMismatch := !isReplicated(tagCount, numSites, tagSet)
 			olockCfgMismatch := !isReplicated(olockCfgCount, numSites, olockConfigSet)
 			sseCfgMismatch := !isReplicated(sseCfgCount, numSites, sseCfgSet)
+			versionCfgMismatch := !isReplicated(versionCfgCount, numSites, versionCfgSet)
 			policyMismatch := !isBktPolicyReplicated(numSites, policies)
 			replCfgMismatch := !isBktReplCfgReplicated(numSites, replCfgs)
 			quotaCfgMismatch := !isBktQuotaCfgReplicated(numSites, quotaCfgs)
@@ -2632,20 +2680,21 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 				}
 				quotaCfgSet := hasBucket && *quotaCfgs[i] != madmin.BucketQuota{}
 				ss := madmin.SRBucketStatsSummary{
-					DeploymentID:           s.DeploymentID,
-					HasBucket:              hasBucket,
-					TagMismatch:            tagMismatch,
-					OLockConfigMismatch:    olockCfgMismatch,
-					SSEConfigMismatch:      sseCfgMismatch,
-					PolicyMismatch:         policyMismatch,
-					ReplicationCfgMismatch: replCfgMismatch,
-					QuotaCfgMismatch:       quotaCfgMismatch,
-					HasReplicationCfg:      s.ReplicationConfig != nil,
-					HasTagsSet:             s.Tags != nil,
-					HasOLockConfigSet:      s.ObjectLockConfig != nil,
-					HasPolicySet:           s.Policy != nil,
-					HasQuotaCfgSet:         quotaCfgSet,
-					HasSSECfgSet:           s.SSEConfig != nil,
+					DeploymentID:             s.DeploymentID,
+					HasBucket:                hasBucket,
+					TagMismatch:              tagMismatch,
+					OLockConfigMismatch:      olockCfgMismatch,
+					SSEConfigMismatch:        sseCfgMismatch,
+					VersioningConfigMismatch: versionCfgMismatch,
+					PolicyMismatch:           policyMismatch,
+					ReplicationCfgMismatch:   replCfgMismatch,
+					QuotaCfgMismatch:         quotaCfgMismatch,
+					HasReplicationCfg:        s.ReplicationConfig != nil,
+					HasTagsSet:               s.Tags != nil,
+					HasOLockConfigSet:        s.ObjectLockConfig != nil,
+					HasPolicySet:             s.Policy != nil,
+					HasQuotaCfgSet:           quotaCfgSet,
+					HasSSECfgSet:             s.SSEConfig != nil,
 				}
 				var m srBucketMetaInfo
 				if len(bucketStats[s.Bucket]) > dIdx {
@@ -2662,6 +2711,9 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 				if !olockCfgMismatch && olockCfgCount == numSites {
 					sum.ReplicatedLockConfig++
 				}
+				if !versionCfgMismatch && versionCfgCount == numSites {
+					sum.ReplicatedVersioningConfig++
+				}
 				if !sseCfgMismatch && sseCfgCount == numSites {
 					sum.ReplicatedSSEConfig++
 				}
@@ -3280,17 +3332,15 @@ func (c *SiteReplicationSys) startHealRoutine(ctx context.Context, objAPI Object
 	for {
 		select {
 		case <-healTimer.C:
-			healTimer.Reset(siteHealTimeInterval)
-
 			c.RLock()
 			enabled := c.enabled
 			c.RUnlock()
-			if !enabled {
-				continue
+			if enabled {
+				c.healIAMSystem(ctx, objAPI) // heal IAM system first
+				c.healBuckets(ctx, objAPI)   // heal buckets subsequently
 			}
+			healTimer.Reset(siteHealTimeInterval)
 
-			c.healIAMSystem(ctx, objAPI) // heal IAM system first
-			c.healBuckets(ctx, objAPI)   // heal buckets subsequently
 		case <-ctx.Done():
 			return
 		}
@@ -3352,6 +3402,7 @@ func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer
 
 	for bucket := range info.BucketStats {
 		c.healCreateMissingBucket(ctx, objAPI, bucket, info)
+		c.healVersioningMetadata(ctx, objAPI, bucket, info)
 		c.healOLockConfigMetadata(ctx, objAPI, bucket, info)
 		c.healSSEMetadata(ctx, objAPI, bucket, info)
 		c.healBucketReplicationConfig(ctx, objAPI, bucket, info)
@@ -3509,7 +3560,6 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 		latestID, latestPeerName string
 		lastUpdate               time.Time
 		latestQuotaConfig        *string
-		latestCfgJSON            []byte
 		latestQuotaConfigBytes   []byte
 	)
 
@@ -3563,7 +3613,7 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 		if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
 			Type:   madmin.SRBucketMetaTypeQuotaConfig,
 			Bucket: bucket,
-			Quota:  latestCfgJSON,
+			Quota:  latestQuotaConfigBytes,
 		}); err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing quota config metadata for peer %s from peer %s : %s",
 				peerName, latestPeerName, err.Error())))
@@ -3572,6 +3622,79 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 	return nil
 }
 
+func (c *SiteReplicationSys) healVersioningMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return nil
+	}
+	var (
+		latestID, latestPeerName string
+		lastUpdate               time.Time
+		latestVersioningConfig   *string
+	)
+
+	bs := info.BucketStats[bucket]
+	for dID, ss := range bs {
+		if lastUpdate.IsZero() {
+			lastUpdate = ss.meta.VersioningConfigUpdatedAt
+			latestID = dID
+			latestVersioningConfig = ss.meta.Versioning
+		}
+		// avoid considering just created buckets as latest. Perhaps this site
+		// just joined cluster replication and yet to be sync'd
+		if ss.meta.CreatedAt.Equal(ss.meta.VersioningConfigUpdatedAt) {
+			continue
+		}
+		if ss.meta.VersioningConfigUpdatedAt.After(lastUpdate) {
+			lastUpdate = ss.meta.VersioningConfigUpdatedAt
+			latestID = dID
+			latestVersioningConfig = ss.meta.Versioning
+		}
+	}
+
+	latestPeerName = info.Sites[latestID].Name
+	var latestVersioningConfigBytes []byte
+	var err error
+	if latestVersioningConfig != nil {
+		latestVersioningConfigBytes, err = base64.StdEncoding.DecodeString(*latestVersioningConfig)
+		if err != nil {
+			return err
+		}
+	}
+
+	for dID, bStatus := range bs {
+		if !bStatus.VersioningConfigMismatch {
+			continue
+		}
+		if isBucketMetadataEqual(latestVersioningConfig, bStatus.meta.Versioning) {
+			continue
+		}
+		if dID == globalDeploymentID {
+			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, latestVersioningConfigBytes); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Error healing sse metadata from peer site %s : %w", latestPeerName, err))
+			}
+			continue
+		}
+
+		admClient, err := c.getAdminClient(ctx, dID)
+		if err != nil {
+			return wrapSRErr(err)
+		}
+		peerName := info.Sites[dID].Name
+		err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
+			Type:       madmin.SRBucketMetaTypeVersionConfig,
+			Bucket:     bucket,
+			Versioning: latestVersioningConfig,
+		})
+		if err != nil {
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing versioning config metadata for peer %s from peer %s : %s",
+				peerName, latestPeerName, err.Error())))
+		}
+	}
+	return nil
+}
+
 func (c *SiteReplicationSys) healSSEMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
 	c.RLock()
 	defer c.RUnlock()

cmd/sts-handlers.go

@@ -284,8 +284,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 				ParentUser:   cred.ParentUser,
 			},
 		}); err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
+			logger.LogIf(ctx, err)
 		}
 	}
 
@@ -383,7 +382,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 			policyName = globalIAMSys.CurrentPolicies(policies)
 		}
 
-		if globalPolicyOPA == nil {
+		if globalAuthZPlugin == nil {
 			if !ok {
 				writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
 					fmt.Errorf("%s claim missing from the JWT token, credentials will not be generated", iamPolicyClaimNameOpenID()))
@@ -479,8 +478,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 			ParentPolicyMapping: policyName,
 		},
 	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+		logger.LogIf(ctx, err)
 	}
 
 	var encodedSuccessResponse []byte
@@ -593,7 +591,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 
 	// Check if this user or their groups have a policy applied.
 	ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, false, groupDistNames...)
-	if len(ldapPolicies) == 0 && globalPolicyOPA == nil {
+	if len(ldapPolicies) == 0 && globalAuthZPlugin == nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
 			fmt.Errorf("expecting a policy to be set for user `%s` or one of their groups: `%s` - rejecting this request",
 				ldapUserDN, strings.Join(groupDistNames, "`,`")))
@@ -649,8 +647,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 			ParentUser:   cred.ParentUser,
 		},
 	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+		logger.LogIf(ctx, err)
 	}
 
 	ldapIdentityResponse := &AssumeRoleWithLDAPResponse{
@@ -810,8 +807,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 			ParentPolicyMapping: policyName,
 		},
 	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+		logger.LogIf(ctx, err)
 	}
 
 	response := new(AssumeRoleWithCertificateResponse)

cmd/sts-handlers_test.go

@@ -356,30 +356,9 @@ const (
 )
 
 func TestIAMWithLDAPServerSuite(t *testing.T) {
-	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
-		// Init and run test on Erasure backend.
-		{serverType: "Erasure", signer: signerV4},
-		// Init and run test on ErasureSet backend.
-		{serverType: "ErasureSet", signer: signerV4},
-	}
-	testCases := []*TestSuiteIAM{}
-	for _, bt := range baseTestCases {
-		testCases = append(testCases,
-			newTestSuiteIAM(bt, false),
-			newTestSuiteIAM(bt, true),
-		)
-	}
-	for i, testCase := range testCases {
-		etcdStr := ""
-		if testCase.withEtcdBackend {
-			etcdStr = " (with etcd backend)"
-		}
+	for i, testCase := range iamTestSuites {
 		t.Run(
-			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
 			func(t *testing.T) {
 				c := &check{t, testCase.serverType}
 				suite := testCase
@@ -1070,30 +1049,9 @@ func (s *TestSuiteIAM) SetUpOpenID(c *check, serverAddr string, rolePolicy strin
 }
 
 func TestIAMWithOpenIDServerSuite(t *testing.T) {
-	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
-		// Init and run test on Erasure backend.
-		{serverType: "Erasure", signer: signerV4},
-		// Init and run test on ErasureSet backend.
-		{serverType: "ErasureSet", signer: signerV4},
-	}
-	testCases := []*TestSuiteIAM{}
-	for _, bt := range baseTestCases {
-		testCases = append(testCases,
-			newTestSuiteIAM(bt, false),
-			newTestSuiteIAM(bt, true),
-		)
-	}
-	for i, testCase := range testCases {
-		etcdStr := ""
-		if testCase.withEtcdBackend {
-			etcdStr = " (with etcd backend)"
-		}
+	for i, testCase := range iamTestSuites {
 		t.Run(
-			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
 			func(t *testing.T) {
 				c := &check{t, testCase.serverType}
 				suite := testCase
@@ -1115,30 +1073,9 @@ func TestIAMWithOpenIDServerSuite(t *testing.T) {
 }
 
 func TestIAMWithOpenIDWithRolePolicyServerSuite(t *testing.T) {
-	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
-		// Init and run test on Erasure backend.
-		{serverType: "Erasure", signer: signerV4},
-		// Init and run test on ErasureSet backend.
-		{serverType: "ErasureSet", signer: signerV4},
-	}
-	testCases := []*TestSuiteIAM{}
-	for _, bt := range baseTestCases {
-		testCases = append(testCases,
-			newTestSuiteIAM(bt, false),
-			newTestSuiteIAM(bt, true),
-		)
-	}
-	for i, testCase := range testCases {
-		etcdStr := ""
-		if testCase.withEtcdBackend {
-			etcdStr = " (with etcd backend)"
-		}
+	for i, testCase := range iamTestSuites {
 		t.Run(
-			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
 			func(t *testing.T) {
 				c := &check{t, testCase.serverType}
 				suite := testCase
@@ -1311,29 +1248,6 @@ func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicy(c *check) {
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket)
 }
 
-// List of all IAM test suites (i.e. test server configuration combinations)
-// common to tests.
-var iamTestSuites = func() []*TestSuiteIAM {
-	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
-		// Init and run test on Erasure backend.
-		{serverType: "Erasure", signer: signerV4},
-		// Init and run test on ErasureSet backend.
-		{serverType: "ErasureSet", signer: signerV4},
-	}
-	testCases := []*TestSuiteIAM{}
-	for _, bt := range baseTestCases {
-		testCases = append(testCases,
-			newTestSuiteIAM(bt, false),
-			newTestSuiteIAM(bt, true),
-		)
-	}
-	return testCases
-}()
-
 func TestIAMWithOpenIDMultipleConfigsValidation(t *testing.T) {
 	openIDServer := os.Getenv(EnvTestOpenIDServer)
 	openIDServer2 := os.Getenv(EnvTestOpenIDServer2)
@@ -1399,3 +1313,194 @@ func TestIAMWithOpenIDWithMultipleRolesServerSuite(t *testing.T) {
 		)
 	}
 }
+
+// Access Management Plugin tests
+func TestIAM_AMPWithOpenIDWithMultipleRolesServerSuite(t *testing.T) {
+	openIDServer := os.Getenv(EnvTestOpenIDServer)
+	openIDServer2 := os.Getenv(EnvTestOpenIDServer2)
+	if openIDServer == "" || openIDServer2 == "" {
+		t.Skip("Skipping OpenID test as enough OpenID servers are not provided.")
+	}
+	testApps := testClientApps
+
+	rolePolicies := []string{
+		"consoleAdmin",
+		"readwrite",
+	}
+
+	for i, testCase := range iamTestSuites {
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
+			func(t *testing.T) {
+				c := &check{t, testCase.serverType}
+				suite := testCase
+
+				suite.SetUpSuite(c)
+				defer suite.TearDownSuite(c)
+
+				err := suite.SetUpOpenIDs(c, testApps, rolePolicies)
+				if err != nil {
+					c.Fatalf("Error setting up openid providers for tests: %v", err)
+				}
+
+				suite.SetUpAccMgmtPlugin(c)
+
+				suite.TestOpenIDSTSWithRolePolicyUnderAMP(c, testRoleARNs[0], testRoleMap[testRoleARNs[0]])
+				suite.TestOpenIDSTSWithRolePolicyUnderAMP(c, testRoleARNs[1], testRoleMap[testRoleARNs[1]])
+				suite.TestOpenIDServiceAccWithRolePolicyUnderAMP(c)
+			},
+		)
+	}
+}
+
+func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicyUnderAMP(c *check, roleARN string, clientApp OpenIDClientAppParams) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket create error: %v", err)
+	}
+
+	// Generate web identity JWT by interacting with OpenID IDP.
+	token, err := MockOpenIDTestUserInteraction(ctx, clientApp, "dillon@example.io", "dillon")
+	if err != nil {
+		c.Fatalf("mock user err: %v", err)
+	}
+
+	// Generate STS credential.
+	webID := cr.STSWebIdentity{
+		Client:      s.TestSuiteCommon.client,
+		STSEndpoint: s.endPoint,
+		GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) {
+			return &cr.WebIdentityToken{
+				Token: token,
+			}, nil
+		},
+		RoleARN: roleARN,
+	}
+
+	value, err := webID.Retrieve()
+	if err != nil {
+		c.Fatalf("Expected to generate STS creds, got err: %#v", err)
+	}
+	// fmt.Printf("value: %#v\n", value)
+
+	minioClient, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("Error initializing client: %v", err)
+	}
+
+	// Validate that the client from sts creds can access the bucket.
+	c.mustListObjects(ctx, minioClient, bucket)
+
+	// Validate that the client from STS creds cannot upload any object as
+	// it is denied by the plugin.
+	c.mustNotUpload(ctx, minioClient, bucket)
+}
+
+func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicyUnderAMP(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket create error: %v", err)
+	}
+
+	// Generate web identity STS token by interacting with OpenID IDP.
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
+	if err != nil {
+		c.Fatalf("mock user err: %v", err)
+	}
+
+	webID := cr.STSWebIdentity{
+		Client:      s.TestSuiteCommon.client,
+		STSEndpoint: s.endPoint,
+		GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) {
+			return &cr.WebIdentityToken{
+				Token: token,
+			}, nil
+		},
+		RoleARN: testRoleARN,
+	}
+
+	value, err := webID.Retrieve()
+	if err != nil {
+		c.Fatalf("Expected to generate STS creds, got err: %#v", err)
+	}
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Create svc acc
+	cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient)
+
+	// 1. Check that svc account appears in listing
+	c.assertSvcAccAppearsInListing(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey)
+
+	// 2. Check that svc account info can be queried
+	c.assertSvcAccInfoQueryable(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey, true)
+
+	// 3. Check S3 access
+	c.assertSvcAccS3Access(ctx, s, cr, bucket)
+	// 3.1 Validate that the client from STS creds cannot upload any object as
+	// it is denied by the plugin.
+	c.mustNotUpload(ctx, s.getUserClient(c, cr.AccessKey, cr.SecretKey, ""), bucket)
+
+	// Check that session policies do not apply - as policy enforcement is
+	// delegated to plugin.
+	{
+		svcAK, svcSK := mustGenerateCredentials(c)
+
+		// This policy does not allow listing objects.
+		policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+		cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+			Policy:     policyBytes,
+			TargetUser: value.AccessKeyID,
+			AccessKey:  svcAK,
+			SecretKey:  svcSK,
+		})
+		if err != nil {
+			c.Fatalf("Unable to create svc acc: %v", err)
+		}
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		// Though the attached policy does not allow listing, it will be
+		// ignored because the plugin allows it.
+		c.mustListObjects(ctx, svcClient, bucket)
+	}
+
+	// 4. Check that service account's secret key and account status can be
+	// updated.
+	c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, value.AccessKeyID, bucket)
+
+	// 5. Check that service account can be deleted.
+	c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket)
+}

cmd/test-utils_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bufio"
 	"bytes"
 	"context"
@@ -46,6 +47,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strconv"
@@ -53,6 +55,7 @@ import (
 	"sync"
 	"testing"
 	"time"
+	"unsafe"
 
 	"github.com/fatih/color"
 
@@ -72,6 +75,9 @@ import (
 func TestMain(m *testing.M) {
 	flag.Parse()
 
+	// set to 'true' when testing is invoked
+	globalIsTesting = true
+
 	globalActiveCred = auth.Credentials{
 		AccessKey: auth.DefaultAccessKey,
 		SecretKey: auth.DefaultSecretKey,
@@ -1183,12 +1189,11 @@ func newTestSignedRequestV4(method, urlStr string, contentLength int64, body io.
 	return req, nil
 }
 
-// Function to generate random string for bucket/object names.
-func randString(n int) string {
-	src := rand.NewSource(UTCNow().UnixNano())
+var src = rand.NewSource(time.Now().UnixNano())
 
+func randString(n int) string {
 	b := make([]byte, n)
-	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
+	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
 	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
 		if remain == 0 {
 			cache, remain = src.Int63(), letterIdxMax
@@ -1200,7 +1205,8 @@ func randString(n int) string {
 		cache >>= letterIdxBits
 		remain--
 	}
-	return string(b)
+
+	return *(*string)(unsafe.Pointer(&b))
 }
 
 // generate random object name.
@@ -2319,3 +2325,35 @@ func uploadTestObject(t *testing.T, apiRouter http.Handler, creds auth.Credentia
 		checkRespErr(rec, http.StatusOK)
 	}
 }
+
+// unzip a file into a specific target dir - used to unzip sample data in cmd/testdata/
+func unzipArchive(zipFilePath, targetDir string) error {
+	zipReader, err := zip.OpenReader(zipFilePath)
+	if err != nil {
+		return err
+	}
+	for _, file := range zipReader.Reader.File {
+		zippedFile, err := file.Open()
+		if err != nil {
+			return err
+		}
+		err = func() (err error) {
+			defer zippedFile.Close()
+			extractedFilePath := filepath.Join(targetDir, file.Name)
+			if file.FileInfo().IsDir() {
+				return os.MkdirAll(extractedFilePath, file.Mode())
+			}
+			outputFile, err := os.OpenFile(extractedFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+			if err != nil {
+				return err
+			}
+			defer outputFile.Close()
+			_, err = io.Copy(outputFile, zippedFile)
+			return err
+		}()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/utils.go

@@ -130,6 +130,9 @@ func getWriteQuorum(drive int) int {
 	return quorum
 }
 
+// CloneMSS is an exposed function of cloneMSS for gateway usage.
+var CloneMSS = cloneMSS
+
 // cloneMSS will clone a map[string]string.
 // If input is nil an empty map is returned, not nil.
 func cloneMSS(v map[string]string) map[string]string {
@@ -1019,12 +1022,14 @@ type AuditLogOptions struct {
 	APIName   string
 	Status    string
 	VersionID string
+	Error     string
 }
 
 // sends audit logs for internal subsystem activity
 func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogOptions) {
 	entry := audit.NewEntry(globalDeploymentID)
 	entry.Trigger = opts.Trigger
+	entry.Error = opts.Error
 	entry.API.Name = opts.APIName
 	entry.API.Bucket = bucket
 	entry.API.Object = object
@@ -1033,6 +1038,11 @@ func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogO
 		entry.ReqQuery[xhttp.VersionID] = opts.VersionID
 	}
 	entry.API.Status = opts.Status
+	// Merge tag information if found - this is currently needed for tags
+	// set during decommissioning.
+	if reqInfo := logger.GetReqInfo(ctx); reqInfo != nil {
+		entry.Tags = reqInfo.GetTagsMap()
+	}
 	ctx = logger.SetAuditEntry(ctx, &entry)
 	logger.AuditLog(ctx, nil, nil, nil)
 }

cmd/xl-storage-disk-id-check.go

@@ -517,6 +517,7 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 		atomic.AddUint64(&p.apiCalls[s], 1)
 		p.apiLatencies[s].add(duration)
 
+		paths = append([]string{p.String()}, paths...)
 		if trace {
 			globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " ")))
 		}

cmd/xl-storage.go

@@ -244,7 +244,7 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		diskIndex:  -1,
 	}
 
-	go formatErasureCleanupTmp(s.diskPath) // cleanup any old data.
+	bgFormatErasureCleanupTmp(s.diskPath) // cleanup any old data.
 
 	formatData, formatFi, err := formatErasureMigrate(s.diskPath)
 	if err != nil && !errors.Is(err, os.ErrNotExist) {

cmd/xl-storage_test.go

@@ -959,13 +959,6 @@ func TestXLStorageDeleteFile(t *testing.T) {
 	}
 	defer os.RemoveAll(path)
 
-	// create xlStorage test setup
-	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup()
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-	// removing the disk, used to recreate disk not found error.
-	os.RemoveAll(diskPath)
 	// Setup test environment.
 	if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
 		t.Fatalf("Unable to create volume, %s", err)
@@ -1066,6 +1059,17 @@ func TestXLStorageDeleteFile(t *testing.T) {
 		}
 	}
 
+	// create xlStorage test setup
+	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create xlStorage test setup, %s", err)
+	}
+	// removing the disk, used to recreate disk not found error.
+	err = os.RemoveAll(diskPath)
+	if err != nil {
+		t.Fatalf("Unable to remoe xlStorage diskpath, %s", err)
+	}
+
 	// TestXLStorage for delete on an removed disk.
 	// should fail with disk not found.
 	err = xlStorageDeletedStorage.Delete(context.Background(), "del-vol", "my-file", false)

docs/bucket/replication/README.md

@@ -243,6 +243,26 @@ Replication from a source bucket to multiple destination buckets is supported. F
 
 Note that on the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` after replication succeeds to each of the targets. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle.
 
+### Interaction with extended Bucket Versioning configuration
+When Bucket Versioning with excluded prefixes are configured objects matching these prefixes are excluded from being versioned and replicated.
+
+```
+<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+        <Status>Enabled</Status>
+        <ExcludeFolders>true</ExcludeFolders>
+
+        <ExcludedPrefixes>
+          <Prefix>app1-jobs/*/_temporary/</Prefix>
+        </ExcludedPrefixes>
+        <ExcludedPrefixes>
+          <Prefix>app2-jobs/*/_magic/</Prefix>
+        </ExcludedPrefixes>
+
+        <!-- .. up to 10 prefixes in all -->
+</VersioningConfiguration>
+```
+In the above sample config, objects under prefixes matching any of the `ExcludedPrefixes` glob patterns will neither be versioned nor replicated.
+
 ## Explore Further
 
 - [MinIO Bucket Replication Design](https://github.com/minio/minio/blob/master/docs/bucket/replication/DESIGN.md)

docs/bucket/versioning/README.md

@@ -63,7 +63,38 @@ Similarly to suspend versioning set the configuration with Status set to `Suspen
 </VersioningConfiguration>
 ```
 
-Only users with explicit permissions or the root credential can configure the versioning state of any bucket.
+## MinIO extension to Bucket Versioning
+### Motivation
+**PLEASE READ: This feature is meant for advanced usecases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on https://subnet.min.io.**
+
+Spark/Hadoop workloads which use Hadoop MR Committer v1/v2 algorithm upload objects to a temporary prefix in a bucket. These objects are 'renamed' to a different prefix on Job commit. Object storage admins are forced to configure separate ILM policies to expire these objects and their versions to reclaim space.
+
+### Solution
+To exclude objects under a list of prefix (glob) patterns from being versioned, you can send the following versioning configuration with Status set to `Enabled`.
+
+```
+<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+        <Status>Enabled</Status>
+        <ExcludeFolders>true</ExcludeFolders>
+
+        <ExcludedPrefixes>
+          <Prefix>*/_temporary</Prefix>
+        </ExcludedPrefixes>
+        <ExcludedPrefixes>
+          <Prefix>*/__magic</Prefix>
+        </ExcludedPrefixes>
+        <ExcludedPrefixes>
+          <Prefix>*/_staging</Prefix>
+        </ExcludedPrefixes>
+
+        <!-- .. up to 10 prefixes in all -->
+</VersioningConfiguration>
+```
+
+### Features
+- Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if bucket has replication configured.
+- Objects matching these prefixes will also not leave `null` delete markers, dramatically reduces namespace pollution while keeping the benefits of replication.
+- Users with explicit permissions or the root credential can configure the versioning state of any bucket.
 
 ## Examples of enabling bucket versioning using MinIO Java SDK
 

docs/debugging/hash-set/main.go

@@ -18,11 +18,16 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/binary"
 	"flag"
 	"fmt"
 	"hash/crc32"
+	"io/ioutil"
 	"log"
+	"os"
+	"strings"
 
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
@@ -64,22 +69,22 @@ func hashOrder(key string, cardinality int) []int {
 }
 
 var (
-	object, deploymentID string
-	setCount, shards     int
+	file, object, deploymentID, prefix string
+	setCount, shards                   int
+	verbose                            bool
 )
 
 func main() {
+	flag.StringVar(&file, "file", "", "Read all objects from file, newline separated")
+	flag.StringVar(&prefix, "prefix", "", "Add prefix to all objects")
 	flag.StringVar(&object, "object", "", "Select an object")
 	flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'")
 	flag.IntVar(&setCount, "set-count", 0, "Total set count")
 	flag.IntVar(&shards, "shards", 0, "Total shards count")
+	flag.BoolVar(&verbose, "v", false, "Display all objects")
 
 	flag.Parse()
 
-	if object == "" {
-		log.Fatalln("object name is mandatory")
-	}
-
 	if deploymentID == "" {
 		log.Fatalln("deployment ID is mandatory")
 	}
@@ -88,12 +93,39 @@ func main() {
 		log.Fatalln("set count cannot be zero")
 	}
 
-	if shards == 0 {
-		log.Fatalln("total shards cannot be zero")
-	}
-
 	id := uuid.MustParse(deploymentID)
 
-	fmt.Println("Erasure distribution for the object", hashOrder(object, shards))
-	fmt.Println("Erasure setNumber for the object", sipHashMod(object, setCount, id)+1)
+	if file != "" {
+		distrib := make([][]string, setCount)
+		b, err := ioutil.ReadFile(file)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		b = bytes.ReplaceAll(b, []byte("\r"), []byte{})
+		sc := bufio.NewScanner(bytes.NewBuffer(b))
+		for sc.Scan() {
+			object = strings.TrimSpace(sc.Text())
+			set := sipHashMod(prefix+object, setCount, id)
+			distrib[set] = append(distrib[set], prefix+object)
+		}
+		for set, files := range distrib {
+			fmt.Println("Set:", set+1, "Objects:", len(files))
+			if !verbose {
+				continue
+			}
+			for _, s := range files {
+				fmt.Printf("\t%s\n", s)
+			}
+		}
+		os.Exit(0)
+	}
+
+	if object == "" {
+		log.Fatalln("object name is mandatory")
+	}
+
+	if shards != 0 {
+		fmt.Println("Erasure distribution for the object", hashOrder(prefix+object, shards))
+	}
+	fmt.Println("Erasure setNumber for the object", sipHashMod(prefix+object, setCount, id)+1)
 }

docs/iam/access-management-plugin.md

@@ -0,0 +1,159 @@
+# Access Management Plugin Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+
+MinIO now includes support for using an Access Management Plugin. This is to allow object storage access control to be managed externally via a webhook.
+
+When configured, MinIO sends request and credential details for every API call to an external HTTP(S) endpoint and expects an allow/deny response. MinIO is thus able to delegate access management to an external system, and users are able to use a custom solution instead of S3 standard IAM policies.
+
+Latency sensitive applications may notice an increased latency due to a request to the external plugin upon every authenticated request to MinIO. User are advised to provision their infrastructure such that latency and performance is acceptable.
+
+## Quickstart
+
+To easily try out the feature, run the included demo Access Management Plugin program in this directory:
+
+```sh
+go run access-manager-plugin.go
+```
+
+This program, lets the admin user perform any action and prevents all other users from performing `s3:Put*` operations.
+
+In another terminal start MinIO:
+
+```sh
+export MINIO_CI_CD=1
+export MINIO_ROOT_USER=minio
+export MINIO_ROOT_PASSWORD=minio123
+export MINIO_POLICY_PLUGIN_URL=http://localhost:8080/
+minio server /tmp/disk{1...4}
+```
+
+Now, let's test it out with `mc`:
+
+```sh
+mc alias set myminio http://localhost:9000 minio minio123
+mc ls myminio
+mc mb myminio/test
+mc cp /etc/issue myminio/test
+mc admin user add myminio foo foobar123
+export MC_HOST_foo=http://foo:foobar123@localhost:9000
+mc ls foo
+mc cp /etc/issue myminio/test/issue2
+```
+
+Only the last operation would fail with a permissions error.
+
+## Configuration
+
+Access Management Plugin can be configured with environment variables:
+
+```sh
+$ mc admin config set dminio1 policy_plugin --env
+KEY:
+policy_plugin  enable Access Management Plugin for policy enforcement
+
+ARGS:
+MINIO_POLICY_PLUGIN_URL*        (url)       plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"
+MINIO_POLICY_PLUGIN_AUTH_TOKEN  (string)    authorization token for plugin hook endpoint
+MINIO_POLICY_PLUGIN_COMMENT     (sentence)  optionally add a comment to this setting
+```
+
+## Request and Response
+
+MinIO will make a `POST` request with a JSON body to the given plugin URL. If the auth token parameter is set, it will be sent as an authorization header.
+
+The JSON body structure can be seen from this sample:
+
+<details><summary>Request Body Sample</summary>
+
+```json
+{
+  "input": {
+    "account": "minio",
+    "groups": null,
+    "action": "s3:ListBucket",
+    "bucket": "test",
+    "conditions": {
+      "Authorization": [
+        "AWS4-HMAC-SHA256 Credential=minio/20220507/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=62012db6c47d697620cf6c68f0f45f6e34894589a53ab1faf6dc94338468c78a"
+      ],
+      "CurrentTime": [
+        "2022-05-07T18:31:41Z"
+      ],
+      "Delimiter": [
+        "/"
+      ],
+      "EpochTime": [
+        "1651948301"
+      ],
+      "Prefix": [
+        ""
+      ],
+      "Referer": [
+        ""
+      ],
+      "SecureTransport": [
+        "false"
+      ],
+      "SourceIp": [
+        "127.0.0.1"
+      ],
+      "User-Agent": [
+        "MinIO (linux; amd64) minio-go/v7.0.24 mc/DEVELOPMENT.2022-04-20T23-07-53Z"
+      ],
+      "UserAgent": [
+        "MinIO (linux; amd64) minio-go/v7.0.24 mc/DEVELOPMENT.2022-04-20T23-07-53Z"
+      ],
+      "X-Amz-Content-Sha256": [
+        "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+      ],
+      "X-Amz-Date": [
+        "20220507T183141Z"
+      ],
+      "authType": [
+        "REST-HEADER"
+      ],
+      "principaltype": [
+        "Account"
+      ],
+      "signatureversion": [
+        "AWS4-HMAC-SHA256"
+      ],
+      "userid": [
+        "minio"
+      ],
+      "username": [
+        "minio"
+      ],
+      "versionid": [
+        ""
+      ]
+    },
+    "owner": true,
+    "object": "",
+    "claims": {},
+    "denyOnly": false
+  }
+}
+```
+
+</details>
+
+The response expected by MinIO, is a JSON body with a boolean:
+
+```json
+{
+    "result": true
+}
+```
+
+The following structure is also accepted:
+
+```json
+{
+    "result": {
+        "allow": true
+    }
+}
+```
+
+Any unmentioned JSON object keys in the above are ignored.
+

docs/iam/access-manager-plugin.go

@@ -0,0 +1,84 @@
+//go:build ignore
+// +build ignore
+
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+)
+
+func writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusBadRequest)
+	json.NewEncoder(w).Encode(map[string]string{
+		"error": fmt.Sprintf("%v", err),
+	})
+}
+
+type Result struct {
+	Result bool `json:"result"`
+}
+
+func mainHandler(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponse(w, err)
+		return
+	}
+
+	reqMap := make(map[string]interface{})
+	err = json.Unmarshal(body, &reqMap)
+	if err != nil {
+		writeErrorResponse(w, err)
+		return
+	}
+
+	// fmt.Printf("request: %#v\n", reqMap)
+
+	m := reqMap["input"].(map[string]interface{})
+	accountValue := m["account"].(string)
+	actionValue := m["action"].(string)
+
+	// Allow user `minio` to perform any action.
+	var res Result
+	if accountValue == "minio" {
+		res.Result = true
+	} else {
+		// All other users may not perform any `s3:Put*` operations.
+		res.Result = true
+		if strings.HasPrefix(actionValue, "s3:Put") {
+			res.Result = false
+		}
+	}
+	fmt.Printf("account: %v | action: %v | allowed: %v\n", accountValue, actionValue, res.Result)
+	json.NewEncoder(w).Encode(res)
+	return
+}
+
+func main() {
+	http.HandleFunc("/", mainHandler)
+
+	log.Print("Listening on :8080")
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

docs/iam/opa.md

@@ -0,0 +1,81 @@
+# OPA Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+OPA is a lightweight general-purpose policy engine that can be co-located with MinIO server, in this document we talk about how to use OPA HTTP API to authorize requests. It can be used with any type of credentials (STS based like OpenID or LDAP, regular IAM users or service accounts).
+
+OPA is enabled through MinIO's Access Management Plugin feature.
+
+## Get started
+
+### 1. Start OPA in a container
+
+```sh
+podman run -it \
+    --name opa \
+    --publish 8181:8181 \
+    docker.io/openpolicyagent/opa:0.40.0-rootless \
+       run --server \
+           --log-format=json-pretty \
+           --log-level=debug \
+           --set=decision_logs.console=true
+```
+
+### 2. Create a sample OPA Policy
+
+In another terminal, create a policy that allows root user all access and for all other users denies `PutObject`:
+```sh
+cat > example.rego <<EOF
+package httpapi.authz
+
+import input
+
+default allow = false
+
+# Allow the root user to perform any action.
+allow {
+ input.owner == true
+}
+
+# All other users may do anything other than call PutObject
+allow {
+ input.action != "s3:PutObject"
+ input.owner == false
+}
+EOF
+```
+
+Then load the policy via OPA's REST API.
+```
+curl -X PUT --data-binary @example.rego \
+  localhost:8181/v1/policies/putobject
+```
+
+### 4. Setup MinIO with OPA
+
+Set the `MINIO_POLICY_PLUGIN_URL` as the endpoint that MinIO should send authorization requests to. Then start the server.
+
+```sh
+export MINIO_POLICY_PLUGIN_URL=http://localhost:8181/v1/data/httpapi/authz/allow
+export MINIO_CI_CD=1
+export MINIO_ROOT_USER=minio
+export MINIO_ROOT_PASSWORD=minio123
+minio server /mnt/data
+```
+
+### 5. Test with a regular IAM user
+
+Ensure that `mc` is installed and the configured with the above server with the alias `myminio`.
+
+```sh
+# 1. Create a bucket and a user, and upload a file. These operations will succeed.
+mc mb myminio/test
+mc admin user add myminio foo foobar123
+mc cp /etc/issue myminio/test/
+
+# 2. Now access the server as user `foo`. These operations will also succeed.
+export MC_HOST_foo=http://foo:foobar123@localhost:9000
+mc ls foo/test
+mc cat foo/test/issue
+
+# 3. Attempt to upload an object as user `foo` - this will fail with a permissions error.
+mc cp /etc/issue myminio/test/issue2
+```
+

docs/metrics/healthcheck/README.md

@@ -40,7 +40,7 @@ readinessProbe:
 
 ### Cluster-writeable probe
 
-This probe is not useful in almost all cases, this is meant for administrators to see if write quorum is available in any given cluster. The reply is '200 OK' if cluster has write quorum if not it returns '503 Service Unavailable'.
+The reply is '200 OK' if cluster has write quorum if not it returns '503 Service Unavailable'.
 
 ```
 curl http://minio1:9001/minio/health/cluster
@@ -59,7 +59,7 @@ Date: Tue, 21 Jul 2020 00:36:14 GMT
 
 ### Cluster-readable probe
 
-This probe is not useful in almost all cases, this is meant for administrators to see if read quorum is available in any given cluster. The reply is '200 OK' if cluster has read quorum if not it returns '503 Service Unavailable'.
+The reply is '200 OK' if cluster has read quorum if not it returns '503 Service Unavailable'.
 
 ```
 curl http://minio1:9001/minio/health/cluster/read

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-04-30T22-23-53Z
+  image: quay.io/minio/minio:RELEASE.2022-05-23T18-45-11Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb v1.0.29
-	github.com/coredns/coredns v1.9.0
+	github.com/coredns/coredns v1.9.2
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dchest/siphash v1.2.1
@@ -21,11 +21,11 @@ require (
 	github.com/dswarbrick/smart v0.0.0-20190505152634-909a45200d6d
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eclipse/paho.mqtt.golang v1.3.5
-	github.com/elastic/go-elasticsearch/v7 v7.12.0
+	github.com/elastic/go-elasticsearch/v7 v7.17.1
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
 	github.com/fraugster/parquet-go v0.10.0
-	github.com/go-ldap/ldap/v3 v3.2.4
+	github.com/go-ldap/ldap/v3 v3.4.3
 	github.com/go-openapi/loads v0.21.1
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang-jwt/jwt/v4 v4.4.1
@@ -41,15 +41,15 @@ require (
 	github.com/klauspost/readahead v1.4.0
 	github.com/klauspost/reedsolomon v1.9.15
 	github.com/lib/pq v1.10.4
-	github.com/miekg/dns v1.1.46
+	github.com/miekg/dns v1.1.48
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.16.2
+	github.com/minio/console v0.17.2
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.3.6
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.19.2
-	github.com/minio/madmin-go v1.3.12
-	github.com/minio/minio-go/v7 v7.0.24
+	github.com/minio/madmin-go v1.3.14
+	github.com/minio/minio-go/v7 v7.0.27
 	github.com/minio/pkg v1.1.23
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
@@ -79,24 +79,24 @@ require (
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yargevad/filepathx v1.0.0
 	github.com/zeebo/xxh3 v1.0.0
-	go.etcd.io/etcd/api/v3 v3.5.2
-	go.etcd.io/etcd/client/v3 v3.5.2
+	go.etcd.io/etcd/api/v3 v3.5.4
+	go.etcd.io/etcd/client/v3 v3.5.4
 	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.21.0
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
+	golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba
 	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
-	google.golang.org/api v0.67.0
+	google.golang.org/api v0.78.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	cloud.google.com/go v0.100.2 // indirect
-	cloud.google.com/go/compute v0.1.0 // indirect
+	cloud.google.com/go/compute v1.6.1 // indirect
 	cloud.google.com/go/iam v0.2.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e // indirect
 	github.com/apache/thrift v0.15.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -118,7 +118,7 @@ require (
 	github.com/frankban/quicktest v1.14.0 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/errors v0.20.2 // indirect
@@ -137,7 +137,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -165,7 +165,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/minio/colorjson v1.0.2 // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20220419155441-cc4ff3a0cc82 // indirect
+	github.com/minio/mc v0.0.0-20220512134321-aa60a8db1e4d // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -184,7 +184,8 @@ require (
 	github.com/posener/complete v1.2.3 // indirect
 	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
 	github.com/pquerna/cachecontrol v0.1.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/common v0.34.0 // indirect
+	github.com/prometheus/prom2json v1.3.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
@@ -200,22 +201,22 @@ require (
 	github.com/unrolled/secure v1.10.0 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
 	go.mongodb.org/mongo-driver v1.8.4 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/goleak v1.1.12 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.8 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a // indirect
-	google.golang.org/grpc v1.44.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect
+	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect

go.sum

@@ -34,8 +34,12 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v0.1.0 h1:rSUBvAyVwNJ5uQCKNJFMwPtTvJkfN38b6Pvb9zZoqJ8=
 cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
+cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
+cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
+cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
+cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc=
+cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
@@ -58,7 +62,7 @@ github.com/Azure/azure-storage-blob-go v0.10.0 h1:evCwGreYo3XLeBV4vSxLbLiYb6e0Sz
 github.com/Azure/azure-storage-blob-go v0.10.0/go.mod h1:ep1edmW+kNQx4UfWM9heESNmQdijykocJ0YOxmMX8SE=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE=
+github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A=
 github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
 github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
 github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ=
@@ -72,8 +76,8 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L
 github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
-github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
-github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e h1:ZU22z/2YRFLyf/P4ZwUYSdNCWsMEI0VeyrFoI2rAhJQ=
+github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -155,12 +159,13 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/coredns/coredns v1.9.0 h1:M1EF1uups4CYcQGb1z8A97mfoq4BYCw3+xCYcJkOSDc=
-github.com/coredns/coredns v1.9.0/go.mod h1:czzy6Ofs15Mzn1PXpWoplBCZxoWdGoQUInL9uPSiYME=
+github.com/coredns/coredns v1.9.2 h1:r1uPYQ/HKQq8zoQ3NP2V4k1hxb3Yw2xN9AXcXzofh6U=
+github.com/coredns/coredns v1.9.2/go.mod h1:U44W7RM94WPp8soWjsm8g08oOQ1A6D1xu4VyCYJ79cc=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM=
@@ -204,8 +209,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y=
 github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
-github.com/elastic/go-elasticsearch/v7 v7.12.0 h1:j4tvcMrZJLp39L2NYvBb7f+lHKPqPHSL3nvB8+/DV+s=
-github.com/elastic/go-elasticsearch/v7 v7.12.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
+github.com/elastic/go-elasticsearch/v7 v7.17.1 h1:49mHcHx7lpCL8cW1aioEwSEVKQF3s+Igi4Ye/QTWwmk=
+github.com/elastic/go-elasticsearch/v7 v7.17.1/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -214,6 +219,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
+github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
@@ -238,19 +244,21 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo
 github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 h1:QqwPZCwh/k1uYqq6uXSb9TRDhTkfQbO80v8zhnIe5zM=
 github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1/go.mod h1:Az6Jt+M5idSED2YPGtwnfJV0kXohgdCBPmHGSYc1r04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=
-github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
+github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-ldap/ldap/v3 v3.2.4 h1:PFavAq2xTgzo/loE8qNXcQaofAaqIpI4WgaLdv+1l3E=
-github.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-ldap/ldap/v3 v3.4.3 h1:JCKUtJPIcyOuG7ctGabLKMgIlKnGumD/iGjuWeEruDI=
+github.com/go-ldap/ldap/v3 v3.4.3/go.mod h1:7LdHfVt6iIOESVEe3Bs4Jp2sHEKgDeduAhgM1/f9qmo=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
@@ -374,8 +382,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -409,8 +418,10 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
+github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI=
+github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
@@ -593,15 +604,15 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
-github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ=
+github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ=
 github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.2 h1:Em3IM68MTm3h+Oxa0nxrV9VQqDgbxvC5iq5A+pqzDeI=
 github.com/minio/colorjson v1.0.2/go.mod h1:JWxcL2n8T8JVf+NY6awl6kn5nK49aAzHOeQEM33dL0k=
-github.com/minio/console v0.16.2 h1:+8kU5b6NzHL+JB0YpbLlCSu5CxGCAoJtPGNy8kDJSTY=
-github.com/minio/console v0.16.2/go.mod h1:qchncyp1uZpt4u5JfixD0+s0utmdutZlQlxCotUg9eo=
+github.com/minio/console v0.17.2 h1:1S6yYqWAEumwyUiHZM/GGYuzmePes4tsXYzQPqVNr7Y=
+github.com/minio/console v0.17.2/go.mod h1:yfF8zcgJtD+KPPFHHD1PMpFbhMnQRY8C7jmsrHBdoRU=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dperf v0.3.6 h1:EAe/OEEuFRGRbcGsfV0+2KsQkrmTvh51UOoCd/h4DDI=
@@ -614,16 +625,16 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
 github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM=
 github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM=
 github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
-github.com/minio/madmin-go v1.3.12 h1:7SmK/KtT7+d3hn3VcYBqI/c4yETfXV9gRT1j+g/U1jE=
-github.com/minio/madmin-go v1.3.12/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
-github.com/minio/mc v0.0.0-20220419155441-cc4ff3a0cc82 h1:CiTaWFwpxzjd7A3sUQ0xZEX8sWfZh3/k2qbxuPip05s=
-github.com/minio/mc v0.0.0-20220419155441-cc4ff3a0cc82/go.mod h1:h6VCl43/2AUA3RP1GWUVMqcUiXq2NWJ4+dSei+ibf70=
+github.com/minio/madmin-go v1.3.14 h1:9f9ZylP5Yn/TcplE/wowsBjb+Czt2+/NRCa2IqpNLcI=
+github.com/minio/madmin-go v1.3.14/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
+github.com/minio/mc v0.0.0-20220512134321-aa60a8db1e4d h1:txmSSDiVFG69Hp/6Yjg5azKl96ObYL3pBQPW0i2uHIs=
+github.com/minio/mc v0.0.0-20220512134321-aa60a8db1e4d/go.mod h1:g9jrk4AQ3yLaxDJzb5D+ww6sGiDC0w1k88LUH5lDR7M=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
-github.com/minio/minio-go/v7 v7.0.24 h1:HPlHiET6L5gIgrHRaw1xFo1OaN4bEP/082asWh3WJtI=
-github.com/minio/minio-go/v7 v7.0.24/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg=
+github.com/minio/minio-go/v7 v7.0.27 h1:yJCvm78B+2+ll1PqO9eSD1as6Ibw3IYnnD8PyBEB2zo=
+github.com/minio/minio-go/v7 v7.0.27/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg=
 github.com/minio/pkg v1.1.20/go.mod h1:Xo7LQshlxGa9shKwJ7NzQbgW4s8T/Wc1cOStR/eUiMY=
 github.com/minio/pkg v1.1.23 h1:CJSoPslQCWZW3z3T79+pv9dVBDCQEK3ipiwXcoAtzY0=
 github.com/minio/pkg v1.1.23/go.mod h1:z9PfmEI804KFkF6eY4LoGe8IDVvTCsYGVuaf58Dr0WI=
@@ -737,6 +748,7 @@ github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -748,8 +760,10 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7q
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
+github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE=
+github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -757,6 +771,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/prom2json v1.3.1 h1:OogL5hsrJpLPz3jZ4LPz4sJRTtADzViCNRQoqrzUQvk=
+github.com/prometheus/prom2json v1.3.1/go.mod h1:A8Oy9aiQx4wrJY9ya1i4nHOySGmkVp5EO0aU1iSJR+g=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 h1:crs4rrYnQqsZpz/EtjezHGCu13e+3W9eqj0MzIYXir0=
@@ -883,14 +899,17 @@ github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+r
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/api/v3 v3.5.2 h1:tXok5yLlKyuQ/SXSjtqHc4uzNaMqZi2XsoSPr/LlJXI=
 go.etcd.io/etcd/api/v3 v3.5.2/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
+go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc=
+go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/pkg/v3 v3.5.2 h1:4hzqQ6hIb3blLyQ8usCU4h3NghkqcsohEQ3o3VetYxE=
 go.etcd.io/etcd/client/pkg/v3 v3.5.2/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg=
+go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
-go.etcd.io/etcd/client/v3 v3.5.2 h1:WdnejrUtQC4nCxK0/dLTMqKOB+U5TP/2Ya0BJL+1otA=
 go.etcd.io/etcd/client/v3 v3.5.2/go.mod h1:kOOaWFFgHygyT0WlSmL8TJiXmMysO/nNUlEsSsN6W4o=
+go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4=
+go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY=
 go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
@@ -929,7 +948,6 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -939,6 +957,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1026,8 +1045,11 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220421235706-1d1ef9303861 h1:yssD99+7tqHWO5Gwh81phT+67hg+KttniBr6UnEXOY8=
-golang.org/x/net v0.0.0-20220421235706-1d1ef9303861/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1045,6 +1067,8 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
+golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE=
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1146,8 +1170,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba h1:AyHWHCBVlIYI5rgEM3o+1PLd0sLPcIAoaUckGQMaWtw=
+golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1240,8 +1268,9 @@ golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
+golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1274,8 +1303,13 @@ google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv
 google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
 google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.67.0 h1:lYaaLa+x3VVUhtosaK9xihwQ9H9KRa557REHwwZ2orM=
 google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
+google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
+google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
+google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
+google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
+google.golang.org/api v0.78.0 h1:5ewPyCwP43C3i8B6C2Kb+eVAevbnke2xR8VbcSWjS4I=
+google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1346,8 +1380,17 @@ google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a h1:uqouglH745GoGeZ1YFZbPBiu961tgi/9Qm5jaorajjQ=
-google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
+google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1374,8 +1417,10 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
+google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1389,8 +1434,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 4.0.1
-appVersion: RELEASE.2022-04-30T22-23-53Z
+version: 4.0.2
+appVersion: RELEASE.2022-05-08T23-50-31Z
 keywords:
   - minio
   - storage

helm/minio/templates/pvc.yaml

@@ -10,6 +10,10 @@ metadata:
     chart: {{ template "minio.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
+{{- if .Values.persistence.annotations }}
+  annotations:
+{{ toYaml .Values.persistence.annotations | trimSuffix "\n" | indent 4 }}
+{{- end }}
 spec:
   accessModes:
     - {{ .Values.persistence.accessMode | quote }}

helm/minio/templates/statefulset.yaml

@@ -184,6 +184,10 @@ spec:
     {{- range $diskId := until $drivesPerNode}}
     - metadata:
         name: export-{{ $diskId }}
+      {{- if $.Values.persistence.annotations }}
+        annotations:
+{{ toYaml $.Values.persistence.annotations | trimSuffix "\n" | indent 10 }}
+      {{- end }}
       spec:
         accessModes: [ {{ $accessMode | quote }} ]
         {{- if $storageClass }}
@@ -196,6 +200,10 @@ spec:
   {{- else }}
     - metadata:
         name: export
+      {{- if $.Values.persistence.annotations }}
+        annotations:
+{{ toYaml $.Values.persistence.annotations | trimSuffix "\n" | indent 10 }}
+      {{- end }}
       spec:
         accessModes: [ {{ $accessMode | quote }} ]
         {{- if $storageClass }}

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2022-04-30T22-23-53Z
+  tag: RELEASE.2022-05-08T23-50-31Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2022-04-16T21-11-21Z
+  tag: RELEASE.2022-05-09T04-08-26Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed or gateway.

index.yaml

@@ -1,9 +1,31 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2022-05-08T23-50-31Z
+    created: "2022-05-08T21:25:26.244555391-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-4.0.2.tgz
+    version: 4.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-04-30T22-23-53Z
-    created: "2022-05-01T23:09:59.936663591-07:00"
+    created: "2022-05-08T21:25:26.243390363-07:00"
     description: Multi-Cloud Object Storage
     digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f
     home: https://min.io
@@ -25,7 +47,7 @@ entries:
     version: 4.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-04-26T01-20-24Z
-    created: "2022-05-01T23:09:59.935421243-07:00"
+    created: "2022-05-08T21:25:26.242212758-07:00"
     description: Multi-Cloud Object Storage
     digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2
     home: https://min.io
@@ -47,7 +69,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-04-16T04-26-02Z
-    created: "2022-05-01T23:09:59.934140158-07:00"
+    created: "2022-05-08T21:25:26.240545805-07:00"
     description: Multi-Cloud Object Storage
     digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1
     home: https://min.io
@@ -69,7 +91,7 @@ entries:
     version: 3.6.6
   - apiVersion: v1
     appVersion: RELEASE.2022-04-12T06-55-35Z
-    created: "2022-05-01T23:09:59.932999424-07:00"
+    created: "2022-05-08T21:25:26.239512723-07:00"
     description: Multi-Cloud Object Storage
     digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650
     home: https://min.io
@@ -91,7 +113,7 @@ entries:
     version: 3.6.5
   - apiVersion: v1
     appVersion: RELEASE.2022-04-09T15-09-52Z
-    created: "2022-05-01T23:09:59.931131991-07:00"
+    created: "2022-05-08T21:25:26.238351228-07:00"
     description: Multi-Cloud Object Storage
     digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06
     home: https://min.io
@@ -113,7 +135,7 @@ entries:
     version: 3.6.4
   - apiVersion: v1
     appVersion: RELEASE.2022-03-24T00-43-44Z
-    created: "2022-05-01T23:09:59.930070603-07:00"
+    created: "2022-05-08T21:25:26.237246981-07:00"
     description: Multi-Cloud Object Storage
     digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75
     home: https://min.io
@@ -135,7 +157,7 @@ entries:
     version: 3.6.3
   - apiVersion: v1
     appVersion: RELEASE.2022-03-17T06-34-49Z
-    created: "2022-05-01T23:09:59.929091086-07:00"
+    created: "2022-05-08T21:25:26.236215859-07:00"
     description: Multi-Cloud Object Storage
     digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495
     home: https://min.io
@@ -157,7 +179,7 @@ entries:
     version: 3.6.2
   - apiVersion: v1
     appVersion: RELEASE.2022-03-14T18-25-24Z
-    created: "2022-05-01T23:09:59.928125748-07:00"
+    created: "2022-05-08T21:25:26.234958967-07:00"
     description: Multi-Cloud Object Storage
     digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
     home: https://min.io
@@ -179,7 +201,7 @@ entries:
     version: 3.6.1
   - apiVersion: v1
     appVersion: RELEASE.2022-03-11T23-57-45Z
-    created: "2022-05-01T23:09:59.927114778-07:00"
+    created: "2022-05-08T21:25:26.233594073-07:00"
     description: Multi-Cloud Object Storage
     digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
     home: https://min.io
@@ -201,7 +223,7 @@ entries:
     version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2022-05-01T23:09:59.925934296-07:00"
+    created: "2022-05-08T21:25:26.232562936-07:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -223,7 +245,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2022-05-01T23:09:59.924286758-07:00"
+    created: "2022-05-08T21:25:26.231606075-07:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -245,7 +267,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-05-01T23:09:59.923230192-07:00"
+    created: "2022-05-08T21:25:26.230596862-07:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -267,7 +289,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-05-01T23:09:59.922111049-07:00"
+    created: "2022-05-08T21:25:26.229423141-07:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -289,7 +311,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2022-05-01T23:09:59.920999838-07:00"
+    created: "2022-05-08T21:25:26.227897178-07:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -311,7 +333,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-05-01T23:09:59.919974275-07:00"
+    created: "2022-05-08T21:25:26.226954506-07:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -333,7 +355,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-05-01T23:09:59.91884654-07:00"
+    created: "2022-05-08T21:25:26.225876966-07:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -355,7 +377,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2022-05-01T23:09:59.917444946-07:00"
+    created: "2022-05-08T21:25:26.224865273-07:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -377,7 +399,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-05-01T23:09:59.91659217-07:00"
+    created: "2022-05-08T21:25:26.223903225-07:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -399,7 +421,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-05-01T23:09:59.915742579-07:00"
+    created: "2022-05-08T21:25:26.222921131-07:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -421,7 +443,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2022-05-01T23:09:59.914794786-07:00"
+    created: "2022-05-08T21:25:26.221513799-07:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -443,7 +465,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2022-05-01T23:09:59.913811538-07:00"
+    created: "2022-05-08T21:25:26.220630353-07:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -465,7 +487,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2022-05-01T23:09:59.912821403-07:00"
+    created: "2022-05-08T21:25:26.219782489-07:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -487,7 +509,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2022-05-01T23:09:59.910956135-07:00"
+    created: "2022-05-08T21:25:26.218795387-07:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -509,7 +531,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2022-05-01T23:09:59.910011754-07:00"
+    created: "2022-05-08T21:25:26.21781401-07:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -531,7 +553,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-05-01T23:09:59.90906256-07:00"
+    created: "2022-05-08T21:25:26.216839576-07:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -553,7 +575,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-05-01T23:09:59.908135027-07:00"
+    created: "2022-05-08T21:25:26.215665829-07:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -575,7 +597,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-05-01T23:09:59.907204524-07:00"
+    created: "2022-05-08T21:25:26.214399445-07:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -597,7 +619,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2022-05-01T23:09:59.906221935-07:00"
+    created: "2022-05-08T21:25:26.213435021-07:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -619,7 +641,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-05-01T23:09:59.90523117-07:00"
+    created: "2022-05-08T21:25:26.212414242-07:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -641,7 +663,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-05-01T23:09:59.90363811-07:00"
+    created: "2022-05-08T21:25:26.211524422-07:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -663,7 +685,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-05-01T23:09:59.902729063-07:00"
+    created: "2022-05-08T21:25:26.210673691-07:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -685,7 +707,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-05-01T23:09:59.901878389-07:00"
+    created: "2022-05-08T21:25:26.209655034-07:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -707,7 +729,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-05-01T23:09:59.901114135-07:00"
+    created: "2022-05-08T21:25:26.208126229-07:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -729,7 +751,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2022-05-01T23:09:59.900218836-07:00"
+    created: "2022-05-08T21:25:26.207167417-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -751,7 +773,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2022-05-01T23:09:59.899267726-07:00"
+    created: "2022-05-08T21:25:26.206164982-07:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -773,7 +795,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2022-05-01T23:09:59.898166583-07:00"
+    created: "2022-05-08T21:25:26.20518876-07:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -795,7 +817,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2022-05-01T23:09:59.896799823-07:00"
+    created: "2022-05-08T21:25:26.204294327-07:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -817,7 +839,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-05-01T23:09:59.896024917-07:00"
+    created: "2022-05-08T21:25:26.203396556-07:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -839,7 +861,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-05-01T23:09:59.895145107-07:00"
+    created: "2022-05-08T21:25:26.202073601-07:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -861,7 +883,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-05-01T23:09:59.894215332-07:00"
+    created: "2022-05-08T21:25:26.201130221-07:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -883,7 +905,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-05-01T23:09:59.893261465-07:00"
+    created: "2022-05-08T21:25:26.200331387-07:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -905,7 +927,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-05-01T23:09:59.892296344-07:00"
+    created: "2022-05-08T21:25:26.199580255-07:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -927,7 +949,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-05-01T23:09:59.89066167-07:00"
+    created: "2022-05-08T21:25:26.198663024-07:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -949,7 +971,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-05-01T23:09:59.889747734-07:00"
+    created: "2022-05-08T21:25:26.197713528-07:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -971,7 +993,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2022-05-01T23:09:59.88882503-07:00"
+    created: "2022-05-08T21:25:26.196740595-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -993,7 +1015,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-05-01T23:09:59.887899656-07:00"
+    created: "2022-05-08T21:25:26.195299581-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -1015,7 +1037,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-05-01T23:09:59.886929472-07:00"
+    created: "2022-05-08T21:25:26.194324215-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -1037,7 +1059,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-05-01T23:09:59.886006418-07:00"
+    created: "2022-05-08T21:25:26.193395544-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -1057,7 +1079,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-05-01T23:09:59.885044474-07:00"
+    created: "2022-05-08T21:25:26.1925295-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -1077,7 +1099,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-05-01T23:09:59.883151516-07:00"
+    created: "2022-05-08T21:25:26.191649488-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -1097,7 +1119,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-05-01T23:09:59.88221234-07:00"
+    created: "2022-05-08T21:25:26.190728818-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -1117,7 +1139,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-05-01T23:09:59.8812642-07:00"
+    created: "2022-05-08T21:25:26.189718576-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -1137,7 +1159,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-05-01T23:09:59.880357381-07:00"
+    created: "2022-05-08T21:25:26.188215525-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -1157,7 +1179,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-05-01T23:09:59.879368392-07:00"
+    created: "2022-05-08T21:25:26.187255927-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -1177,7 +1199,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2022-05-01T23:09:59.878473433-07:00"
+    created: "2022-05-08T21:25:26.186327631-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -1195,4 +1217,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2022-05-01T23:09:59.877340128-07:00"
+generated: "2022-05-08T21:25:26.185246246-07:00"

internal/bucket/lifecycle/filter.go

@@ -20,6 +20,9 @@ package lifecycle
 import (
 	"encoding/xml"
 	"io"
+	"log"
+
+	"github.com/minio/minio-go/v7/pkg/tags"
 )
 
 var errInvalidFilter = Errorf("Filter must have exactly one of Prefix, Tag, or And specified")
@@ -36,8 +39,9 @@ type Filter struct {
 
 	Tag    Tag
 	tagSet bool
+
 	// Caching tags, only once
-	cachedTags []string
+	cachedTags map[string]string
 }
 
 // MarshalXML - produces the xml representation of the Filter struct
@@ -150,27 +154,42 @@ func (f Filter) Validate() error {
 
 // TestTags tests if the object tags satisfy the Filter tags requirement,
 // it returns true if there is no tags in the underlying Filter.
-func (f Filter) TestTags(tags []string) bool {
+func (f Filter) TestTags(userTags string) bool {
 	if f.cachedTags == nil {
-		tags := make([]string, 0)
+		cache := make(map[string]string)
 		for _, t := range append(f.And.Tags, f.Tag) {
 			if !t.IsEmpty() {
-				tags = append(tags, t.String())
+				cache[t.Key] = t.Value
 			}
 		}
-		f.cachedTags = tags
+		f.cachedTags = cache
 	}
-	for _, ct := range f.cachedTags {
-		foundTag := false
-		for _, t := range tags {
-			if ct == t {
-				foundTag = true
-				break
-			}
-		}
-		if !foundTag {
-			return false
+
+	// This filter does not have any tags, always return true
+	if len(f.cachedTags) == 0 {
+		return true
+	}
+
+	parsedTags, err := tags.ParseObjectTags(userTags)
+	if err != nil {
+		log.Printf("Unexpected object tag found: `%s`\n", userTags)
+		return false
+	}
+	tagsMap := parsedTags.ToMap()
+
+	// This filter has tags configured but this object
+	// does not have any tag, skip this object
+	if len(tagsMap) == 0 {
+		return false
+	}
+
+	// Both filter and object have tags, find a match,
+	// skip this object otherwise
+	for k, cv := range f.cachedTags {
+		v, ok := tagsMap[k]
+		if ok && v == cv {
+			return true
 		}
 	}
-	return true
+	return false
 }

internal/bucket/lifecycle/lifecycle.go

@@ -259,7 +259,7 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
 			continue
 		}
 
-		if rule.Filter.TestTags(strings.Split(obj.UserTags, "&")) {
+		if rule.Filter.TestTags(obj.UserTags) {
 			rules = append(rules, rule)
 		}
 		if !rule.Transition.IsNull() {

internal/bucket/lifecycle/lifecycle_test.go

@@ -325,6 +325,14 @@ func TestComputeActions(t *testing.T) {
 			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
 			expectedAction: DeleteAction,
 		},
+		// Should remove (Tags with encoded chars)
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><And><Tag><Key>factory</Key><Value>true</Value></Tag><Tag><Key>store forever</Key><Value>false</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			objectName:     "fooobject",
+			objectTags:     "store+forever=false&factory=true",
+			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
+			expectedAction: DeleteAction,
+		},
 
 		// Should not remove (Tags don't match)
 		{

internal/bucket/replication/replication.go

@@ -124,6 +124,7 @@ const (
 	HealReplicationType
 	ExistingObjectReplicationType
 	ResyncReplicationType
+	AllReplicationType
 )
 
 // Valid returns true if replication type is set
@@ -148,7 +149,7 @@ type ObjectOpts struct {
 // FilterActionableRules returns the rules actions that need to be executed
 // after evaluating prefix/tag filtering
 func (c Config) FilterActionableRules(obj ObjectOpts) []Rule {
-	if obj.Name == "" && obj.OpType != ResyncReplicationType {
+	if obj.Name == "" && !(obj.OpType == ResyncReplicationType || obj.OpType == AllReplicationType) {
 		return nil
 	}
 	var rules []Rule
@@ -160,8 +161,8 @@ func (c Config) FilterActionableRules(obj ObjectOpts) []Rule {
 		if obj.TargetArn != "" && rule.Destination.ARN != obj.TargetArn && c.RoleArn != obj.TargetArn {
 			continue
 		}
-		// Ignore other object level and prefix filters for resyncing target
-		if obj.OpType == ResyncReplicationType {
+		// Ignore other object level and prefix filters for resyncing target/listing bucket targets
+		if obj.OpType == ResyncReplicationType || obj.OpType == AllReplicationType {
 			rules = append(rules, rule)
 			continue
 		}

internal/bucket/versioning/versioning.go

@@ -20,6 +20,9 @@ package versioning
 import (
 	"encoding/xml"
 	"io"
+	"strings"
+
+	"github.com/minio/pkg/wildcard"
 )
 
 // State - enabled/disabled/suspended states
@@ -33,12 +36,26 @@ const (
 	Suspended State = "Suspended"
 )
 
+var (
+	errExcludedPrefixNotSupported = Errorf("excluded prefixes extension supported only when versioning is enabled")
+	errTooManyExcludedPrefixes    = Errorf("too many excluded prefixes")
+)
+
+// ExcludedPrefix - holds individual prefixes excluded from being versioned.
+type ExcludedPrefix struct {
+	Prefix string
+}
+
 // Versioning - Configuration for bucket versioning.
 type Versioning struct {
 	XMLNS   string   `xml:"xmlns,attr,omitempty"`
 	XMLName xml.Name `xml:"VersioningConfiguration"`
 	// MFADelete State    `xml:"MFADelete,omitempty"` // not supported yet.
 	Status State `xml:"Status,omitempty"`
+	// MinIO extension - allows selective, prefix-level versioning exclusion.
+	// Requires versioning to be enabled
+	ExcludedPrefixes []ExcludedPrefix `xml:",omitempty"`
+	ExcludeFolders   bool             `xml:",omitempty"`
 }
 
 // Validate - validates the versioning configuration
@@ -50,7 +67,16 @@ func (v Versioning) Validate() error {
 	// 	return Errorf("unsupported MFADelete state %s", v.MFADelete)
 	// }
 	switch v.Status {
-	case Enabled, Suspended:
+	case Enabled:
+		const maxExcludedPrefixes = 10
+		if len(v.ExcludedPrefixes) > maxExcludedPrefixes {
+			return errTooManyExcludedPrefixes
+		}
+
+	case Suspended:
+		if len(v.ExcludedPrefixes) > 0 {
+			return errExcludedPrefixNotSupported
+		}
 	default:
 		return Errorf("unsupported Versioning status %s", v.Status)
 	}
@@ -62,11 +88,67 @@ func (v Versioning) Enabled() bool {
 	return v.Status == Enabled
 }
 
+// PrefixEnabled - returns true if versioning is enabled at the bucket and given
+// prefix, false otherwise.
+func (v Versioning) PrefixEnabled(prefix string) bool {
+	if v.Status != Enabled {
+		return false
+	}
+
+	if prefix == "" {
+		return true
+	}
+	if v.ExcludeFolders && strings.HasSuffix(prefix, "/") {
+		return false
+	}
+
+	for _, sprefix := range v.ExcludedPrefixes {
+		// Note: all excluded prefix patterns end with `/` (See Validate)
+		sprefix.Prefix += "*"
+
+		if matched := wildcard.MatchSimple(sprefix.Prefix, prefix); matched {
+			return false
+		}
+	}
+	return true
+}
+
 // Suspended - returns true if versioning is suspended
 func (v Versioning) Suspended() bool {
 	return v.Status == Suspended
 }
 
+// PrefixSuspended - returns true if versioning is suspended at the bucket level
+// or suspended on the given prefix.
+func (v Versioning) PrefixSuspended(prefix string) bool {
+	if v.Status == Suspended {
+		return true
+	}
+	if v.Status == Enabled {
+		if prefix == "" {
+			return false
+		}
+		if v.ExcludeFolders && strings.HasSuffix(prefix, "/") {
+			return true
+		}
+
+		for _, sprefix := range v.ExcludedPrefixes {
+			// Note: all excluded prefix patterns end with `/` (See Validate)
+			sprefix.Prefix += "*"
+			if matched := wildcard.MatchSimple(sprefix.Prefix, prefix); matched {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// PrefixesExcluded returns true if v contains one or more excluded object
+// prefixes or if ExcludeFolders is true.
+func (v Versioning) PrefixesExcluded() bool {
+	return len(v.ExcludedPrefixes) > 0 || v.ExcludeFolders
+}
+
 // ParseConfig - parses data in given reader to VersioningConfiguration.
 func ParseConfig(reader io.Reader) (*Versioning, error) {
 	var v Versioning

internal/bucket/versioning/versioning_test.go

@@ -0,0 +1,241 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package versioning
+
+import (
+	"encoding/xml"
+	"strings"
+	"testing"
+)
+
+func TestParseConfig(t *testing.T) {
+	testcases := []struct {
+		input            string
+		err              error
+		excludedPrefixes []string
+		excludeFolders   bool
+	}{
+		{
+			input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                                  <Status>Enabled</Status>
+                                </VersioningConfiguration>`,
+			err: nil,
+		},
+		{
+			input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                                  <Status>Enabled</Status>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_temporary/</Prefix>
+                                  </ExcludedPrefixes>
+                                </VersioningConfiguration>`,
+			err:              nil,
+			excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"},
+		},
+		{
+			input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                                  <Status>Suspended</Status>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging</Prefix>
+                                  </ExcludedPrefixes>
+                                </VersioningConfiguration>`,
+			err: errExcludedPrefixNotSupported,
+		},
+		{
+			input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                                  <Status>Enabled</Status>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/ab/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/cd/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/ef/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/gh/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/ij/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/kl/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/mn/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/op/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/qr/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/st/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/uv/</Prefix>
+                                  </ExcludedPrefixes>
+                                </VersioningConfiguration>`,
+			err: errTooManyExcludedPrefixes,
+		},
+		{
+			input: `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+                                  <Status>Enabled</Status>
+                                  <ExcludeFolders>true</ExcludeFolders>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_staging/</Prefix>
+                                  </ExcludedPrefixes>
+                                  <ExcludedPrefixes>
+                                    <Prefix>path/to/my/workload/_temporary/</Prefix>
+                                  </ExcludedPrefixes>
+                                </VersioningConfiguration>`,
+			err:              nil,
+			excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"},
+			excludeFolders:   true,
+		},
+	}
+
+	for i, tc := range testcases {
+		var v *Versioning
+		var err error
+		v, err = ParseConfig(strings.NewReader(tc.input))
+		if tc.err != err {
+			t.Fatalf("Test %d: expected %v but got %v", i+1, tc.err, err)
+		}
+		if err != nil {
+			if tc.err == nil {
+				t.Fatalf("Test %d: failed due to %v", i+1, err)
+			}
+		} else {
+			if err := v.Validate(); tc.err != err {
+				t.Fatalf("Test %d: validation failed due to %v", i+1, err)
+			}
+			if len(tc.excludedPrefixes) > 0 {
+				var mismatch bool
+				if len(v.ExcludedPrefixes) != len(tc.excludedPrefixes) {
+					t.Fatalf("Test %d: Expected length of excluded prefixes %d but got %d", i+1, len(tc.excludedPrefixes), len(v.ExcludedPrefixes))
+				}
+				var i int
+				var eprefix string
+				for i, eprefix = range tc.excludedPrefixes {
+					if eprefix != v.ExcludedPrefixes[i].Prefix {
+						mismatch = true
+						break
+					}
+				}
+				if mismatch {
+					t.Fatalf("Test %d: Expected excluded prefix %s but got %s", i+1, tc.excludedPrefixes[i], v.ExcludedPrefixes[i].Prefix)
+				}
+			}
+			if tc.excludeFolders != v.ExcludeFolders {
+				t.Fatalf("Test %d: Expected ExcludeFoldersr=%v but got %v", i+1, tc.excludeFolders, v.ExcludeFolders)
+			}
+		}
+	}
+}
+
+func TestMarshalXML(t *testing.T) {
+	// Validates if Versioning with no excluded prefixes omits
+	// ExcludedPrefixes tags
+	v := Versioning{
+		Status: Enabled,
+	}
+	buf, err := xml.Marshal(v)
+	if err != nil {
+		t.Fatalf("Failed to marshal %v: %v", v, err)
+	}
+
+	str := string(buf)
+	if strings.Contains(str, "ExcludedPrefixes") {
+		t.Fatalf("XML shouldn't contain ExcludedPrefixes tag - %s", str)
+	}
+}
+
+func TestVersioningZero(t *testing.T) {
+	var v Versioning
+	if v.Enabled() {
+		t.Fatalf("Expected to be disabled but got enabled")
+	}
+	if v.Suspended() {
+		t.Fatalf("Expected to be disabled but got suspended")
+	}
+}
+
+func TestExcludeFolders(t *testing.T) {
+	v := Versioning{
+		Status:         Enabled,
+		ExcludeFolders: true,
+	}
+	testPrefixes := []string{"jobs/output/_temporary/", "jobs/output/", "jobs/"}
+	for i, prefix := range testPrefixes {
+		if v.PrefixEnabled(prefix) || !v.PrefixSuspended(prefix) {
+			t.Fatalf("Test %d: Expected versioning to be excluded for %s", i+1, prefix)
+		}
+	}
+
+	// Test applicability for regular objects
+	if prefix := "prefix-1/obj-1"; !v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) {
+		t.Fatalf("Expected versioning to be enabled for %s", prefix)
+	}
+
+	// Test when ExcludeFolders is disabled
+	v.ExcludeFolders = false
+	for i, prefix := range testPrefixes {
+		if !v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) {
+			t.Fatalf("Test %d: Expected versioning to be enabled for %s", i+1, prefix)
+		}
+	}
+}
+
+func TestExcludedPrefixesMatch(t *testing.T) {
+	v := Versioning{
+		Status:           Enabled,
+		ExcludedPrefixes: []ExcludedPrefix{{"*/_temporary/"}},
+	}
+
+	if err := v.Validate(); err != nil {
+		t.Fatalf("Invalid test versioning config %v: %v", v, err)
+	}
+	tests := []struct {
+		prefix   string
+		excluded bool
+	}{
+		{
+			prefix:   "app1-jobs/output/_temporary/attempt1/data.csv",
+			excluded: true,
+		},
+		{
+			prefix:   "app1-jobs/output/final/attempt1/data.csv",
+			excluded: false,
+		},
+	}
+
+	for i, test := range tests {
+		if v.PrefixSuspended(test.prefix) != test.excluded {
+			if test.excluded {
+				t.Fatalf("Test %d: Expected prefix %s to be excluded from versioning", i+1, test.prefix)
+			} else {
+				t.Fatalf("Test %d: Expected prefix %s to have versioning enabled", i+1, test.prefix)
+			}
+		}
+	}
+}

internal/config/compress/help.go

@@ -26,6 +26,13 @@ var (
 	}
 
 	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         config.Enable,
+			Description: "Enable or disable object compression",
+			Type:        "on|off",
+			Optional:    true,
+			Sensitive:   false,
+		},
 		config.HelpKV{
 			Key:         Extensions,
 			Description: `comma separated file extensions` + defaultHelpPostfix(Extensions),

internal/config/config.go

@@ -69,6 +69,7 @@ const (
 const (
 	CredentialsSubSys    = "credentials"
 	PolicyOPASubSys      = "policy_opa"
+	PolicyPluginSubSys   = "policy_plugin"
 	IdentityOpenIDSubSys = "identity_openid"
 	IdentityLDAPSubSys   = "identity_ldap"
 	IdentityTLSSubSys    = "identity_tls"
@@ -141,6 +142,7 @@ var SubSystems = set.CreateStringSet(
 	AuditWebhookSubSys,
 	AuditKafkaSubSys,
 	PolicyOPASubSys,
+	PolicyPluginSubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
 	IdentityTLSSubSys,
@@ -183,6 +185,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	StorageClassSubSys,
 	CompressionSubSys,
 	PolicyOPASubSys,
+	PolicyPluginSubSys,
 	IdentityLDAPSubSys,
 	IdentityTLSSubSys,
 	HealSubSys,

internal/config/identity/ldap/config.go

@@ -62,6 +62,31 @@ type Config struct {
 	rootCAs           *x509.CertPool
 }
 
+// Clone returns a cloned copy of LDAP config.
+func (l *Config) Clone() Config {
+	if l == nil {
+		return Config{}
+	}
+	cfg := Config{
+		Enabled:                   l.Enabled,
+		ServerAddr:                l.ServerAddr,
+		UserDNSearchBaseDistName:  l.UserDNSearchBaseDistName,
+		UserDNSearchBaseDistNames: l.UserDNSearchBaseDistNames,
+		UserDNSearchFilter:        l.UserDNSearchFilter,
+		GroupSearchBaseDistName:   l.GroupSearchBaseDistName,
+		GroupSearchBaseDistNames:  l.GroupSearchBaseDistNames,
+		GroupSearchFilter:         l.GroupSearchFilter,
+		LookupBindDN:              l.LookupBindDN,
+		LookupBindPassword:        l.LookupBindPassword,
+		stsExpiryDuration:         l.stsExpiryDuration,
+		tlsSkipVerify:             l.tlsSkipVerify,
+		serverInsecure:            l.serverInsecure,
+		serverStartTLS:            l.serverStartTLS,
+		rootCAs:                   l.rootCAs,
+	}
+	return cfg
+}
+
 // LDAP keys and envs.
 const (
 	ServerAddr         = "server_addr"

internal/config/identity/openid/help.go

@@ -26,6 +26,13 @@ var (
 	}
 
 	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         config.Enable,
+			Description: "Enable or disable OpenID",
+			Type:        "on|off",
+			Optional:    true,
+			Sensitive:   false,
+		},
 		config.HelpKV{
 			Key:         DisplayName,
 			Description: "Friendly display name for this Provider/App" + defaultHelpPostfix(DisplayName),

internal/config/identity/openid/jwt.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -19,34 +19,21 @@ package openid
 
 import (
 	"crypto"
-	"crypto/sha1"
-	"crypto/sha256"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
-	"strconv"
-	"strings"
 	"sync"
 	"time"
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
-	"github.com/minio/madmin-go"
-	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/identity/openid/provider"
-	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
 )
 
-var errSingleProvider = config.Errorf("Only one OpenID provider can be configured if not using role policy mapping")
-
 type publicKeys struct {
 	*sync.RWMutex
 
@@ -79,185 +66,6 @@ func (pk *publicKeys) get(kid string) crypto.PublicKey {
 	return pk.pkMap[kid]
 }
 
-type providerCfg struct {
-	// Used for user interface like console
-	DisplayName string `json:"displayName,omitempty"`
-
-	JWKS struct {
-		URL *xnet.URL `json:"url"`
-	} `json:"jwks"`
-	URL                *xnet.URL `json:"url,omitempty"`
-	ClaimPrefix        string    `json:"claimPrefix,omitempty"`
-	ClaimName          string    `json:"claimName,omitempty"`
-	ClaimUserinfo      bool      `json:"claimUserInfo,omitempty"`
-	RedirectURI        string    `json:"redirectURI,omitempty"`
-	RedirectURIDynamic bool      `json:"redirectURIDynamic"`
-	DiscoveryDoc       DiscoveryDoc
-	ClientID           string
-	ClientSecret       string
-	RolePolicy         string
-
-	roleArn  arn.ARN
-	provider provider.Provider
-}
-
-// initializeProvider initializes if any additional vendor specific information
-// was provided, initialization will return an error initial login fails.
-func (p *providerCfg) initializeProvider(cfgGet func(string, string) string, transport http.RoundTripper) error {
-	vendor := cfgGet(EnvIdentityOpenIDVendor, Vendor)
-	if vendor == "" {
-		return nil
-	}
-	var err error
-	switch vendor {
-	case keyCloakVendor:
-		adminURL := cfgGet(EnvIdentityOpenIDKeyCloakAdminURL, KeyCloakAdminURL)
-		realm := cfgGet(EnvIdentityOpenIDKeyCloakRealm, KeyCloakRealm)
-		p.provider, err = provider.KeyCloak(
-			provider.WithAdminURL(adminURL),
-			provider.WithOpenIDConfig(provider.DiscoveryDoc(p.DiscoveryDoc)),
-			provider.WithTransport(transport),
-			provider.WithRealm(realm),
-		)
-		return err
-	default:
-		return fmt.Errorf("Unsupport vendor %s", keyCloakVendor)
-	}
-}
-
-// UserInfo returns claims for authenticated user from userInfo endpoint.
-//
-// Some OIDC implementations such as GitLab do not support
-// claims as part of the normal oauth2 flow, instead rely
-// on service providers making calls to IDP to fetch additional
-// claims available from the UserInfo endpoint
-func (p *providerCfg) UserInfo(accessToken string, transport http.RoundTripper) (map[string]interface{}, error) {
-	if p.JWKS.URL == nil || p.JWKS.URL.String() == "" {
-		return nil, errors.New("openid not configured")
-	}
-	client := &http.Client{
-		Transport: transport,
-	}
-
-	req, err := http.NewRequest(http.MethodPost, p.DiscoveryDoc.UserInfoEndpoint, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if accessToken != "" {
-		req.Header.Set("Authorization", "Bearer "+accessToken)
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	defer xhttp.DrainBody(resp.Body)
-	if resp.StatusCode != http.StatusOK {
-		// uncomment this for debugging when needed.
-		// reqBytes, _ := httputil.DumpRequest(req, false)
-		// fmt.Println(string(reqBytes))
-		// respBytes, _ := httputil.DumpResponse(resp, true)
-		// fmt.Println(string(respBytes))
-		return nil, errors.New(resp.Status)
-	}
-
-	dec := json.NewDecoder(resp.Body)
-	claims := map[string]interface{}{}
-
-	if err = dec.Decode(&claims); err != nil {
-		// uncomment this for debugging when needed.
-		// reqBytes, _ := httputil.DumpRequest(req, false)
-		// fmt.Println(string(reqBytes))
-		// respBytes, _ := httputil.DumpResponse(resp, true)
-		// fmt.Println(string(respBytes))
-		return nil, err
-	}
-
-	return claims, nil
-}
-
-// Config - OpenID Config
-// RSA authentication target arguments
-type Config struct {
-	Enabled bool `json:"enabled"`
-
-	// map of roleARN to providerCfg's
-	arnProviderCfgsMap map[arn.ARN]*providerCfg
-
-	// map of config names to providerCfg's
-	ProviderCfgs map[string]*providerCfg
-
-	pubKeys          publicKeys
-	roleArnPolicyMap map[arn.ARN]string
-
-	transport   http.RoundTripper
-	closeRespFn func(io.ReadCloser)
-}
-
-// GetIAMPolicyClaimName - returns the policy claim name for the (at most one)
-// provider configured without a role policy.
-func (r *Config) GetIAMPolicyClaimName() string {
-	pCfg, ok := r.arnProviderCfgsMap[DummyRoleARN]
-	if !ok {
-		return ""
-	}
-	return pCfg.ClaimPrefix + pCfg.ClaimName
-}
-
-// LookupUser lookup userid for the provider
-func (r Config) LookupUser(roleArn, userid string) (provider.User, error) {
-	// Can safely ignore error here as empty or invalid ARNs will not be
-	// mapped.
-	arnVal, _ := arn.Parse(roleArn)
-	pCfg, ok := r.arnProviderCfgsMap[arnVal]
-	if ok {
-		user, err := pCfg.provider.LookupUser(userid)
-		if err != nil && err != provider.ErrAccessTokenExpired {
-			return user, err
-		}
-		if err == provider.ErrAccessTokenExpired {
-			if err = pCfg.provider.LoginWithClientID(pCfg.ClientID, pCfg.ClientSecret); err != nil {
-				return user, err
-			}
-			user, err = pCfg.provider.LookupUser(userid)
-		}
-		return user, err
-	}
-	// Without any specific logic for a provider, all accounts
-	// are always enabled.
-	return provider.User{ID: userid, Enabled: true}, nil
-}
-
-const (
-	keyCloakVendor = "keycloak"
-)
-
-// ProviderEnabled returns true if any vendor specific provider is enabled.
-func (r Config) ProviderEnabled() bool {
-	if !r.Enabled {
-		return false
-	}
-	for _, v := range r.arnProviderCfgsMap {
-		if v.provider != nil {
-			return true
-		}
-	}
-	return false
-}
-
-// GetRoleInfo - returns ARN to policies map if a role policy based openID
-// provider is configured. Otherwise returns nil.
-func (r Config) GetRoleInfo() map[arn.ARN]string {
-	for _, p := range r.arnProviderCfgsMap {
-		if p.RolePolicy != "" {
-			return r.roleArnPolicyMap
-		}
-	}
-	return nil
-}
-
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey(arn arn.ARN) error {
 	pCfg := r.arnProviderCfgsMap[arn]
@@ -280,42 +88,6 @@ func (r *Config) PopulatePublicKey(arn arn.ARN) error {
 	return r.pubKeys.parseAndAdd(resp.Body)
 }
 
-// UnmarshalJSON - decodes JSON data.
-func (r *Config) UnmarshalJSON(data []byte) error {
-	// subtype to avoid recursive call to UnmarshalJSON()
-	type subConfig Config
-	var sr subConfig
-
-	if err := json.Unmarshal(data, &sr); err != nil {
-		return err
-	}
-
-	ar := Config(sr)
-	*r = ar
-	return nil
-}
-
-// GetDefaultExpiration - returns the expiration seconds expected.
-func GetDefaultExpiration(dsecs string) (time.Duration, error) {
-	defaultExpiryDuration := time.Duration(60) * time.Minute // Defaults to 1hr.
-	if dsecs != "" {
-		expirySecs, err := strconv.ParseInt(dsecs, 10, 64)
-		if err != nil {
-			return 0, auth.ErrInvalidDuration
-		}
-
-		// The duration, in seconds, of the role session.
-		// The value can range from 900 seconds (15 minutes)
-		// up to 365 days.
-		if expirySecs < 900 || expirySecs > 31536000 {
-			return 0, auth.ErrInvalidDuration
-		}
-
-		defaultExpiryDuration = time.Duration(expirySecs) * time.Second
-	}
-	return defaultExpiryDuration, nil
-}
-
 // ErrTokenExpired - error token expired
 var (
 	ErrTokenExpired = errors.New("token expired")
@@ -461,86 +233,6 @@ func (r *Config) updateUserinfoClaims(arn arn.ARN, accessToken string, claims ma
 	return nil
 }
 
-// GetSettings - fetches OIDC settings for site-replication related validation.
-// NOTE that region must be populated by caller as this package does not know.
-func (r *Config) GetSettings() madmin.OpenIDSettings {
-	res := madmin.OpenIDSettings{}
-	if !r.Enabled {
-		return res
-	}
-
-	for arn, provCfg := range r.arnProviderCfgsMap {
-		hashedSecret := ""
-		{
-			h := sha256.New()
-			h.Write([]byte(provCfg.ClientSecret))
-			bs := h.Sum(nil)
-			hashedSecret = base64.RawURLEncoding.EncodeToString(bs)
-		}
-		if arn != DummyRoleARN {
-			if res.Roles != nil {
-				res.Roles = make(map[string]madmin.OpenIDProviderSettings)
-			}
-			res.Roles[arn.String()] = madmin.OpenIDProviderSettings{
-				ClaimUserinfoEnabled: provCfg.ClaimUserinfo,
-				RolePolicy:           provCfg.RolePolicy,
-				ClientID:             provCfg.ClientID,
-				HashedClientSecret:   hashedSecret,
-			}
-		} else {
-			res.ClaimProvider = madmin.OpenIDProviderSettings{
-				ClaimUserinfoEnabled: provCfg.ClaimUserinfo,
-				RolePolicy:           provCfg.RolePolicy,
-				ClientID:             provCfg.ClientID,
-				HashedClientSecret:   hashedSecret,
-			}
-		}
-
-	}
-
-	return res
-}
-
-// OpenID keys and envs.
-const (
-	JwksURL       = "jwks_url"
-	ConfigURL     = "config_url"
-	ClaimName     = "claim_name"
-	ClaimUserinfo = "claim_userinfo"
-	ClaimPrefix   = "claim_prefix"
-	ClientID      = "client_id"
-	ClientSecret  = "client_secret"
-	RolePolicy    = "role_policy"
-	DisplayName   = "display_name"
-
-	Vendor             = "vendor"
-	Scopes             = "scopes"
-	RedirectURI        = "redirect_uri"
-	RedirectURIDynamic = "redirect_uri_dynamic"
-
-	// Vendor specific ENV only enabled if the Vendor matches == "vendor"
-	KeyCloakRealm    = "keycloak_realm"
-	KeyCloakAdminURL = "keycloak_admin_url"
-
-	EnvIdentityOpenIDEnable             = "MINIO_IDENTITY_OPENID_ENABLE"
-	EnvIdentityOpenIDVendor             = "MINIO_IDENTITY_OPENID_VENDOR"
-	EnvIdentityOpenIDClientID           = "MINIO_IDENTITY_OPENID_CLIENT_ID"
-	EnvIdentityOpenIDClientSecret       = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
-	EnvIdentityOpenIDURL                = "MINIO_IDENTITY_OPENID_CONFIG_URL"
-	EnvIdentityOpenIDClaimName          = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
-	EnvIdentityOpenIDClaimUserInfo      = "MINIO_IDENTITY_OPENID_CLAIM_USERINFO"
-	EnvIdentityOpenIDClaimPrefix        = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
-	EnvIdentityOpenIDRolePolicy         = "MINIO_IDENTITY_OPENID_ROLE_POLICY"
-	EnvIdentityOpenIDRedirectURI        = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
-	EnvIdentityOpenIDRedirectURIDynamic = "MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC"
-	EnvIdentityOpenIDScopes             = "MINIO_IDENTITY_OPENID_SCOPES"
-	EnvIdentityOpenIDDisplayName        = "MINIO_IDENTITY_OPENID_DISPLAY_NAME"
-
-	// Vendor specific ENVs only enabled if the Vendor matches == "vendor"
-	EnvIdentityOpenIDKeyCloakRealm    = "MINIO_IDENTITY_OPENID_KEYCLOAK_REALM"
-	EnvIdentityOpenIDKeyCloakAdminURL = "MINIO_IDENTITY_OPENID_KEYCLOAK_ADMIN_URL"
-)
-
 // DiscoveryDoc - parses the output from openid-configuration
 // for example https://accounts.google.com/.well-known/openid-configuration
 type DiscoveryDoc struct {
@@ -582,312 +274,3 @@ func parseDiscoveryDoc(u *xnet.URL, transport http.RoundTripper, closeRespFn fun
 	}
 	return d, nil
 }
-
-// DefaultKVS - default config for OpenID config
-var (
-	DefaultKVS = config.KVS{
-		config.KV{
-			Key:   config.Enable,
-			Value: "",
-		},
-		config.KV{
-			Key:   DisplayName,
-			Value: "",
-		},
-		config.KV{
-			Key:   ConfigURL,
-			Value: "",
-		},
-		config.KV{
-			Key:   ClientID,
-			Value: "",
-		},
-		config.KV{
-			Key:   ClientSecret,
-			Value: "",
-		},
-		config.KV{
-			Key:   ClaimName,
-			Value: iampolicy.PolicyName,
-		},
-		config.KV{
-			Key:   ClaimUserinfo,
-			Value: "",
-		},
-		config.KV{
-			Key:   RolePolicy,
-			Value: "",
-		},
-		config.KV{
-			Key:   ClaimPrefix,
-			Value: "",
-		},
-		config.KV{
-			Key:   RedirectURI,
-			Value: "",
-		},
-		config.KV{
-			Key:   RedirectURIDynamic,
-			Value: "off",
-		},
-		config.KV{
-			Key:   Scopes,
-			Value: "",
-		},
-	}
-)
-
-// Enabled returns if configURL is enabled.
-func Enabled(kvs config.KVS) bool {
-	return kvs.Get(ConfigURL) != ""
-}
-
-// DummyRoleARN is used to indicate that the user associated with it was
-// authenticated via policy-claim based OpenID provider.
-var DummyRoleARN = func() arn.ARN {
-	v, err := arn.NewIAMRoleARN("dummy-internal", "")
-	if err != nil {
-		panic("should not happen!")
-	}
-	return v
-}()
-
-// LookupConfig lookup jwks from config, override with any ENVs.
-func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, closeRespFn func(io.ReadCloser), serverRegion string) (c Config, err error) {
-	openIDClientTransport := http.DefaultTransport
-	if transport != nil {
-		openIDClientTransport = transport
-	}
-	c = Config{
-		Enabled:            false,
-		arnProviderCfgsMap: map[arn.ARN]*providerCfg{},
-		ProviderCfgs:       map[string]*providerCfg{},
-		pubKeys: publicKeys{
-			RWMutex: &sync.RWMutex{},
-			pkMap:   map[string]crypto.PublicKey{},
-		},
-		roleArnPolicyMap: map[arn.ARN]string{},
-		transport:        openIDClientTransport,
-		closeRespFn:      closeRespFn,
-	}
-
-	// Make a copy of the config we received so we can mutate it safely.
-	kvsMap2 := make(map[string]config.KVS, len(kvsMap))
-	for k, v := range kvsMap {
-		kvsMap2[k] = v
-	}
-
-	// Add in each configuration name found from environment variables, i.e.
-	// if we see MINIO_IDENTITY_OPENID_CONFIG_URL_2, we add the key "2" to
-	// `kvsMap2` if it does not already exist.
-	envs := env.List(EnvIdentityOpenIDURL + config.Default)
-	for _, k := range envs {
-		cfgName := strings.TrimPrefix(k, EnvIdentityOpenIDURL+config.Default)
-		if cfgName == "" {
-			return c, config.Errorf("Environment variable must have a non-empty config name: %s", k)
-		}
-
-		// It is possible that some variables were specified via config
-		// commands and some variables are intended to be overridden
-		// from the environment, so we ensure that the key is not
-		// overwritten in `kvsMap2` as it may have existing config.
-		if _, ok := kvsMap2[cfgName]; !ok {
-			kvsMap2[cfgName] = DefaultKVS
-		}
-	}
-
-	var (
-		hasLegacyPolicyMapping = false
-		seenClientIDs          = set.NewStringSet()
-	)
-	for cfgName, kvs := range kvsMap2 {
-		// remove this since we have removed support for this already.
-		kvs.Delete(JwksURL)
-
-		if err = config.CheckValidKeys(config.IdentityOpenIDSubSys, kvs, DefaultKVS); err != nil {
-			return c, err
-		}
-
-		getCfgVal := func(envVar, cfgParam string) string {
-			if cfgName != config.Default {
-				envVar += config.Default + cfgName
-			}
-			return env.Get(envVar, kvs.Get(cfgParam))
-		}
-
-		// In the past, when only one openID provider was allowed, there
-		// was no `enable` parameter - the configuration is turned off
-		// by clearing the values. With multiple providers, we support
-		// individually enabling/disabling provider configurations. If
-		// the enable parameter's value is non-empty, we use that
-		// setting, otherwise we treat it as enabled if some important
-		// parameters are non-empty.
-		var (
-			cfgEnableVal        = getCfgVal(EnvIdentityOpenIDEnable, config.Enable)
-			isExplicitlyEnabled = false
-		)
-		if cfgEnableVal != "" {
-			isExplicitlyEnabled = true
-		}
-
-		var enabled bool
-		if isExplicitlyEnabled {
-			enabled, err = config.ParseBool(cfgEnableVal)
-			if err != nil {
-				return c, err
-			}
-			// No need to continue loading if the config is not enabled.
-			if !enabled {
-				continue
-			}
-		}
-
-		p := providerCfg{
-			DisplayName:        getCfgVal(EnvIdentityOpenIDDisplayName, DisplayName),
-			ClaimName:          getCfgVal(EnvIdentityOpenIDClaimName, ClaimName),
-			ClaimUserinfo:      getCfgVal(EnvIdentityOpenIDClaimUserInfo, ClaimUserinfo) == config.EnableOn,
-			ClaimPrefix:        getCfgVal(EnvIdentityOpenIDClaimPrefix, ClaimPrefix),
-			RedirectURI:        getCfgVal(EnvIdentityOpenIDRedirectURI, RedirectURI),
-			RedirectURIDynamic: getCfgVal(EnvIdentityOpenIDRedirectURIDynamic, RedirectURIDynamic) == config.EnableOn,
-			ClientID:           getCfgVal(EnvIdentityOpenIDClientID, ClientID),
-			ClientSecret:       getCfgVal(EnvIdentityOpenIDClientSecret, ClientSecret),
-			RolePolicy:         getCfgVal(EnvIdentityOpenIDRolePolicy, RolePolicy),
-		}
-
-		configURL := getCfgVal(EnvIdentityOpenIDURL, ConfigURL)
-
-		if !isExplicitlyEnabled {
-			enabled = true
-			if p.ClientID == "" && p.ClientSecret == "" && configURL == "" {
-				enabled = false
-			}
-		}
-
-		// No need to continue loading if the config is not enabled.
-		if !enabled {
-			continue
-		}
-
-		// Validate that client ID has not been duplicately specified.
-		if seenClientIDs.Contains(p.ClientID) {
-			return c, config.Errorf("Client ID %s is present with multiple OpenID configurations", p.ClientID)
-		}
-		seenClientIDs.Add(p.ClientID)
-
-		p.URL, err = xnet.ParseHTTPURL(configURL)
-		if err != nil {
-			return c, err
-		}
-		configURLDomain := p.URL.Hostname()
-		p.DiscoveryDoc, err = parseDiscoveryDoc(p.URL, transport, closeRespFn)
-		if err != nil {
-			return c, err
-		}
-
-		if p.ClaimUserinfo && configURL == "" {
-			return c, errors.New("please specify config_url to enable fetching claims from UserInfo endpoint")
-		}
-
-		if scopeList := getCfgVal(EnvIdentityOpenIDScopes, Scopes); scopeList != "" {
-			var scopes []string
-			for _, scope := range strings.Split(scopeList, ",") {
-				scope = strings.TrimSpace(scope)
-				if scope == "" {
-					return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList)
-				}
-				scopes = append(scopes, scope)
-			}
-			// Replace the discovery document scopes by client customized scopes.
-			p.DiscoveryDoc.ScopesSupported = scopes
-		}
-
-		// Check if claim name is the non-default value and role policy is set.
-		if p.ClaimName != iampolicy.PolicyName && p.RolePolicy != "" {
-			// In the unlikely event that the user specifies
-			// `iampolicy.PolicyName` as the claim name explicitly and sets
-			// a role policy, this check is thwarted, but we will be using
-			// the role policy anyway.
-			return c, config.Errorf("Role Policy (=`%s`) and Claim Name (=`%s`) cannot both be set", p.RolePolicy, p.ClaimName)
-		}
-
-		jwksURL := p.DiscoveryDoc.JwksURI
-		if jwksURL == "" {
-			return c, config.Errorf("no JWKS URI found in your provider's discovery doc (config_url=%s)", configURL)
-		}
-
-		p.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL)
-		if err != nil {
-			return c, err
-		}
-
-		if p.RolePolicy != "" {
-			// RolePolicy is validated by IAM System during its
-			// initialization.
-
-			// Generate role ARN as combination of provider domain and
-			// prefix of client ID.
-			domain := configURLDomain
-			if domain == "" {
-				// Attempt to parse the JWKs URI.
-				domain = p.JWKS.URL.Hostname()
-				if domain == "" {
-					return c, config.Errorf("unable to generate a domain from the OpenID config")
-				}
-			}
-			if p.ClientID == "" {
-				return c, config.Errorf("client ID must not be empty")
-			}
-
-			// We set the resource ID of the role arn as a hash of client
-			// ID, so we can get a short roleARN that stays the same on
-			// restart.
-			var resourceID string
-			{
-				h := sha1.New()
-				h.Write([]byte(p.ClientID))
-				bs := h.Sum(nil)
-				resourceID = base64.RawURLEncoding.EncodeToString(bs)
-			}
-			p.roleArn, err = arn.NewIAMRoleARN(resourceID, serverRegion)
-			if err != nil {
-				return c, config.Errorf("unable to generate ARN from the OpenID config: %v", err)
-			}
-
-			c.roleArnPolicyMap[p.roleArn] = p.RolePolicy
-		} else if p.ClaimName == "" {
-			return c, config.Errorf("A role policy or claim name must be specified")
-		}
-
-		if err = p.initializeProvider(getCfgVal, c.transport); err != nil {
-			return c, err
-		}
-
-		arnKey := p.roleArn
-		if p.RolePolicy == "" {
-			arnKey = DummyRoleARN
-			hasLegacyPolicyMapping = true
-			// Ensure that when a JWT policy claim based provider
-			// exists, it is the only one.
-			if _, ok := c.arnProviderCfgsMap[DummyRoleARN]; ok {
-				return c, errSingleProvider
-			}
-		}
-
-		c.arnProviderCfgsMap[arnKey] = &p
-		c.ProviderCfgs[cfgName] = &p
-
-		if err = c.PopulatePublicKey(arnKey); err != nil {
-			return c, err
-		}
-	}
-
-	// Ensure that when a JWT policy claim based provider
-	// exists, it is the only one.
-	if hasLegacyPolicyMapping && len(c.ProviderCfgs) > 1 {
-		return c, errSingleProvider
-	}
-
-	c.Enabled = true
-
-	return c, nil
-}

internal/config/identity/openid/openid.go

@@ -0,0 +1,544 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package openid
+
+import (
+	"crypto"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/arn"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid/provider"
+	"github.com/minio/pkg/env"
+	iampolicy "github.com/minio/pkg/iam/policy"
+	xnet "github.com/minio/pkg/net"
+)
+
+// OpenID keys and envs.
+const (
+	JwksURL       = "jwks_url"
+	ConfigURL     = "config_url"
+	ClaimName     = "claim_name"
+	ClaimUserinfo = "claim_userinfo"
+	ClaimPrefix   = "claim_prefix"
+	ClientID      = "client_id"
+	ClientSecret  = "client_secret"
+	RolePolicy    = "role_policy"
+	DisplayName   = "display_name"
+
+	Vendor             = "vendor"
+	Scopes             = "scopes"
+	RedirectURI        = "redirect_uri"
+	RedirectURIDynamic = "redirect_uri_dynamic"
+
+	// Vendor specific ENV only enabled if the Vendor matches == "vendor"
+	KeyCloakRealm    = "keycloak_realm"
+	KeyCloakAdminURL = "keycloak_admin_url"
+
+	EnvIdentityOpenIDEnable             = "MINIO_IDENTITY_OPENID_ENABLE"
+	EnvIdentityOpenIDVendor             = "MINIO_IDENTITY_OPENID_VENDOR"
+	EnvIdentityOpenIDClientID           = "MINIO_IDENTITY_OPENID_CLIENT_ID"
+	EnvIdentityOpenIDClientSecret       = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
+	EnvIdentityOpenIDURL                = "MINIO_IDENTITY_OPENID_CONFIG_URL"
+	EnvIdentityOpenIDClaimName          = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
+	EnvIdentityOpenIDClaimUserInfo      = "MINIO_IDENTITY_OPENID_CLAIM_USERINFO"
+	EnvIdentityOpenIDClaimPrefix        = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
+	EnvIdentityOpenIDRolePolicy         = "MINIO_IDENTITY_OPENID_ROLE_POLICY"
+	EnvIdentityOpenIDRedirectURI        = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
+	EnvIdentityOpenIDRedirectURIDynamic = "MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC"
+	EnvIdentityOpenIDScopes             = "MINIO_IDENTITY_OPENID_SCOPES"
+	EnvIdentityOpenIDDisplayName        = "MINIO_IDENTITY_OPENID_DISPLAY_NAME"
+
+	// Vendor specific ENVs only enabled if the Vendor matches == "vendor"
+	EnvIdentityOpenIDKeyCloakRealm    = "MINIO_IDENTITY_OPENID_KEYCLOAK_REALM"
+	EnvIdentityOpenIDKeyCloakAdminURL = "MINIO_IDENTITY_OPENID_KEYCLOAK_ADMIN_URL"
+)
+
+// DefaultKVS - default config for OpenID config
+var (
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   config.Enable,
+			Value: "",
+		},
+		config.KV{
+			Key:   DisplayName,
+			Value: "",
+		},
+		config.KV{
+			Key:   ConfigURL,
+			Value: "",
+		},
+		config.KV{
+			Key:   ClientID,
+			Value: "",
+		},
+		config.KV{
+			Key:   ClientSecret,
+			Value: "",
+		},
+		config.KV{
+			Key:   ClaimName,
+			Value: iampolicy.PolicyName,
+		},
+		config.KV{
+			Key:   ClaimUserinfo,
+			Value: "",
+		},
+		config.KV{
+			Key:   RolePolicy,
+			Value: "",
+		},
+		config.KV{
+			Key:   ClaimPrefix,
+			Value: "",
+		},
+		config.KV{
+			Key:   RedirectURI,
+			Value: "",
+		},
+		config.KV{
+			Key:   RedirectURIDynamic,
+			Value: "off",
+		},
+		config.KV{
+			Key:   Scopes,
+			Value: "",
+		},
+	}
+)
+
+var errSingleProvider = config.Errorf("Only one OpenID provider can be configured if not using role policy mapping")
+
+// DummyRoleARN is used to indicate that the user associated with it was
+// authenticated via policy-claim based OpenID provider.
+var DummyRoleARN = func() arn.ARN {
+	v, err := arn.NewIAMRoleARN("dummy-internal", "")
+	if err != nil {
+		panic("should not happen!")
+	}
+	return v
+}()
+
+// Config - OpenID Config
+type Config struct {
+	Enabled bool
+
+	// map of roleARN to providerCfg's
+	arnProviderCfgsMap map[arn.ARN]*providerCfg
+
+	// map of config names to providerCfg's
+	ProviderCfgs map[string]*providerCfg
+
+	pubKeys          publicKeys
+	roleArnPolicyMap map[arn.ARN]string
+
+	transport   http.RoundTripper
+	closeRespFn func(io.ReadCloser)
+}
+
+// Clone returns a cloned copy of OpenID config.
+func (r *Config) Clone() Config {
+	if r == nil {
+		return Config{}
+	}
+	cfg := Config{
+		Enabled:            r.Enabled,
+		arnProviderCfgsMap: make(map[arn.ARN]*providerCfg, len(r.arnProviderCfgsMap)),
+		ProviderCfgs:       make(map[string]*providerCfg, len(r.ProviderCfgs)),
+		pubKeys:            r.pubKeys,
+		roleArnPolicyMap:   make(map[arn.ARN]string, len(r.roleArnPolicyMap)),
+		transport:          r.transport,
+		closeRespFn:        r.closeRespFn,
+	}
+	for k, v := range r.arnProviderCfgsMap {
+		cfg.arnProviderCfgsMap[k] = v
+	}
+	for k, v := range r.ProviderCfgs {
+		cfg.ProviderCfgs[k] = v
+	}
+	for k, v := range r.roleArnPolicyMap {
+		cfg.roleArnPolicyMap[k] = v
+	}
+	return cfg
+}
+
+// LookupConfig lookup jwks from config, override with any ENVs.
+func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, closeRespFn func(io.ReadCloser), serverRegion string) (c Config, err error) {
+	openIDClientTransport := http.DefaultTransport
+	if transport != nil {
+		openIDClientTransport = transport
+	}
+	c = Config{
+		Enabled:            false,
+		arnProviderCfgsMap: map[arn.ARN]*providerCfg{},
+		ProviderCfgs:       map[string]*providerCfg{},
+		pubKeys: publicKeys{
+			RWMutex: &sync.RWMutex{},
+			pkMap:   map[string]crypto.PublicKey{},
+		},
+		roleArnPolicyMap: map[arn.ARN]string{},
+		transport:        openIDClientTransport,
+		closeRespFn:      closeRespFn,
+	}
+
+	// Make a copy of the config we received so we can mutate it safely.
+	kvsMap2 := make(map[string]config.KVS, len(kvsMap))
+	for k, v := range kvsMap {
+		kvsMap2[k] = v
+	}
+
+	// Add in each configuration name found from environment variables, i.e.
+	// if we see MINIO_IDENTITY_OPENID_CONFIG_URL_2, we add the key "2" to
+	// `kvsMap2` if it does not already exist.
+	envs := env.List(EnvIdentityOpenIDURL + config.Default)
+	for _, k := range envs {
+		cfgName := strings.TrimPrefix(k, EnvIdentityOpenIDURL+config.Default)
+		if cfgName == "" {
+			return c, config.Errorf("Environment variable must have a non-empty config name: %s", k)
+		}
+
+		// It is possible that some variables were specified via config
+		// commands and some variables are intended to be overridden
+		// from the environment, so we ensure that the key is not
+		// overwritten in `kvsMap2` as it may have existing config.
+		if _, ok := kvsMap2[cfgName]; !ok {
+			kvsMap2[cfgName] = DefaultKVS
+		}
+	}
+
+	var (
+		hasLegacyPolicyMapping = false
+		seenClientIDs          = set.NewStringSet()
+	)
+	for cfgName, kvs := range kvsMap2 {
+		// remove this since we have removed support for this already.
+		kvs.Delete(JwksURL)
+
+		if err = config.CheckValidKeys(config.IdentityOpenIDSubSys, kvs, DefaultKVS); err != nil {
+			return c, err
+		}
+
+		getCfgVal := func(envVar, cfgParam string) string {
+			if cfgName != config.Default {
+				envVar += config.Default + cfgName
+			}
+			return env.Get(envVar, kvs.Get(cfgParam))
+		}
+
+		// In the past, when only one openID provider was allowed, there
+		// was no `enable` parameter - the configuration is turned off
+		// by clearing the values. With multiple providers, we support
+		// individually enabling/disabling provider configurations. If
+		// the enable parameter's value is non-empty, we use that
+		// setting, otherwise we treat it as enabled if some important
+		// parameters are non-empty.
+		var (
+			cfgEnableVal        = getCfgVal(EnvIdentityOpenIDEnable, config.Enable)
+			isExplicitlyEnabled = false
+		)
+		if cfgEnableVal != "" {
+			isExplicitlyEnabled = true
+		}
+
+		var enabled bool
+		if isExplicitlyEnabled {
+			enabled, err = config.ParseBool(cfgEnableVal)
+			if err != nil {
+				return c, err
+			}
+			// No need to continue loading if the config is not enabled.
+			if !enabled {
+				continue
+			}
+		}
+
+		p := newProviderCfgFromConfig(getCfgVal)
+		configURL := getCfgVal(EnvIdentityOpenIDURL, ConfigURL)
+
+		if !isExplicitlyEnabled {
+			enabled = true
+			if p.ClientID == "" && p.ClientSecret == "" && configURL == "" {
+				enabled = false
+			}
+		}
+
+		// No need to continue loading if the config is not enabled.
+		if !enabled {
+			continue
+		}
+
+		// Validate that client ID has not been duplicately specified.
+		if seenClientIDs.Contains(p.ClientID) {
+			return c, config.Errorf("Client ID %s is present with multiple OpenID configurations", p.ClientID)
+		}
+		seenClientIDs.Add(p.ClientID)
+
+		p.URL, err = xnet.ParseHTTPURL(configURL)
+		if err != nil {
+			return c, err
+		}
+		configURLDomain := p.URL.Hostname()
+		p.DiscoveryDoc, err = parseDiscoveryDoc(p.URL, transport, closeRespFn)
+		if err != nil {
+			return c, err
+		}
+
+		if p.ClaimUserinfo && configURL == "" {
+			return c, errors.New("please specify config_url to enable fetching claims from UserInfo endpoint")
+		}
+
+		if scopeList := getCfgVal(EnvIdentityOpenIDScopes, Scopes); scopeList != "" {
+			var scopes []string
+			for _, scope := range strings.Split(scopeList, ",") {
+				scope = strings.TrimSpace(scope)
+				if scope == "" {
+					return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList)
+				}
+				scopes = append(scopes, scope)
+			}
+			// Replace the discovery document scopes by client customized scopes.
+			p.DiscoveryDoc.ScopesSupported = scopes
+		}
+
+		// Check if claim name is the non-default value and role policy is set.
+		if p.ClaimName != iampolicy.PolicyName && p.RolePolicy != "" {
+			// In the unlikely event that the user specifies
+			// `iampolicy.PolicyName` as the claim name explicitly and sets
+			// a role policy, this check is thwarted, but we will be using
+			// the role policy anyway.
+			return c, config.Errorf("Role Policy (=`%s`) and Claim Name (=`%s`) cannot both be set", p.RolePolicy, p.ClaimName)
+		}
+
+		jwksURL := p.DiscoveryDoc.JwksURI
+		if jwksURL == "" {
+			return c, config.Errorf("no JWKS URI found in your provider's discovery doc (config_url=%s)", configURL)
+		}
+
+		p.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL)
+		if err != nil {
+			return c, err
+		}
+
+		if p.RolePolicy != "" {
+			// RolePolicy is validated by IAM System during its
+			// initialization.
+
+			// Generate role ARN as combination of provider domain and
+			// prefix of client ID.
+			domain := configURLDomain
+			if domain == "" {
+				// Attempt to parse the JWKs URI.
+				domain = p.JWKS.URL.Hostname()
+				if domain == "" {
+					return c, config.Errorf("unable to parse a domain from the OpenID config")
+				}
+			}
+			if p.ClientID == "" {
+				return c, config.Errorf("client ID must not be empty")
+			}
+
+			// We set the resource ID of the role arn as a hash of client
+			// ID, so we can get a short roleARN that stays the same on
+			// restart.
+			var resourceID string
+			{
+				h := sha1.New()
+				h.Write([]byte(p.ClientID))
+				bs := h.Sum(nil)
+				resourceID = base64.RawURLEncoding.EncodeToString(bs)
+			}
+			p.roleArn, err = arn.NewIAMRoleARN(resourceID, serverRegion)
+			if err != nil {
+				return c, config.Errorf("unable to generate ARN from the OpenID config: %v", err)
+			}
+
+			c.roleArnPolicyMap[p.roleArn] = p.RolePolicy
+		} else if p.ClaimName == "" {
+			return c, config.Errorf("A role policy or claim name must be specified")
+		}
+
+		if err = p.initializeProvider(getCfgVal, c.transport); err != nil {
+			return c, err
+		}
+
+		arnKey := p.roleArn
+		if p.RolePolicy == "" {
+			arnKey = DummyRoleARN
+			hasLegacyPolicyMapping = true
+			// Ensure that when a JWT policy claim based provider
+			// exists, it is the only one.
+			if _, ok := c.arnProviderCfgsMap[DummyRoleARN]; ok {
+				return c, errSingleProvider
+			}
+		}
+
+		c.arnProviderCfgsMap[arnKey] = &p
+		c.ProviderCfgs[cfgName] = &p
+
+		if err = c.PopulatePublicKey(arnKey); err != nil {
+			return c, err
+		}
+	}
+
+	// Ensure that when a JWT policy claim based provider
+	// exists, it is the only one.
+	if hasLegacyPolicyMapping && len(c.ProviderCfgs) > 1 {
+		return c, errSingleProvider
+	}
+
+	c.Enabled = true
+
+	return c, nil
+}
+
+// Enabled returns if configURL is enabled.
+func Enabled(kvs config.KVS) bool {
+	return kvs.Get(ConfigURL) != ""
+}
+
+// GetSettings - fetches OIDC settings for site-replication related validation.
+// NOTE that region must be populated by caller as this package does not know.
+func (r *Config) GetSettings() madmin.OpenIDSettings {
+	res := madmin.OpenIDSettings{}
+	if !r.Enabled {
+		return res
+	}
+
+	for arn, provCfg := range r.arnProviderCfgsMap {
+		hashedSecret := ""
+		{
+			h := sha256.New()
+			h.Write([]byte(provCfg.ClientSecret))
+			bs := h.Sum(nil)
+			hashedSecret = base64.RawURLEncoding.EncodeToString(bs)
+		}
+		if arn != DummyRoleARN {
+			if res.Roles != nil {
+				res.Roles = make(map[string]madmin.OpenIDProviderSettings)
+			}
+			res.Roles[arn.String()] = madmin.OpenIDProviderSettings{
+				ClaimUserinfoEnabled: provCfg.ClaimUserinfo,
+				RolePolicy:           provCfg.RolePolicy,
+				ClientID:             provCfg.ClientID,
+				HashedClientSecret:   hashedSecret,
+			}
+		} else {
+			res.ClaimProvider = madmin.OpenIDProviderSettings{
+				ClaimUserinfoEnabled: provCfg.ClaimUserinfo,
+				RolePolicy:           provCfg.RolePolicy,
+				ClientID:             provCfg.ClientID,
+				HashedClientSecret:   hashedSecret,
+			}
+		}
+
+	}
+
+	return res
+}
+
+// GetIAMPolicyClaimName - returns the policy claim name for the (at most one)
+// provider configured without a role policy.
+func (r *Config) GetIAMPolicyClaimName() string {
+	pCfg, ok := r.arnProviderCfgsMap[DummyRoleARN]
+	if !ok {
+		return ""
+	}
+	return pCfg.ClaimPrefix + pCfg.ClaimName
+}
+
+// LookupUser lookup userid for the provider
+func (r Config) LookupUser(roleArn, userid string) (provider.User, error) {
+	// Can safely ignore error here as empty or invalid ARNs will not be
+	// mapped.
+	arnVal, _ := arn.Parse(roleArn)
+	pCfg, ok := r.arnProviderCfgsMap[arnVal]
+	if ok {
+		user, err := pCfg.provider.LookupUser(userid)
+		if err != nil && err != provider.ErrAccessTokenExpired {
+			return user, err
+		}
+		if err == provider.ErrAccessTokenExpired {
+			if err = pCfg.provider.LoginWithClientID(pCfg.ClientID, pCfg.ClientSecret); err != nil {
+				return user, err
+			}
+			user, err = pCfg.provider.LookupUser(userid)
+		}
+		return user, err
+	}
+	// Without any specific logic for a provider, all accounts
+	// are always enabled.
+	return provider.User{ID: userid, Enabled: true}, nil
+}
+
+// ProviderEnabled returns true if any vendor specific provider is enabled.
+func (r Config) ProviderEnabled() bool {
+	if !r.Enabled {
+		return false
+	}
+	for _, v := range r.arnProviderCfgsMap {
+		if v.provider != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// GetRoleInfo - returns ARN to policies map if a role policy based openID
+// provider is configured. Otherwise returns nil.
+func (r Config) GetRoleInfo() map[arn.ARN]string {
+	for _, p := range r.arnProviderCfgsMap {
+		if p.RolePolicy != "" {
+			return r.roleArnPolicyMap
+		}
+	}
+	return nil
+}
+
+// GetDefaultExpiration - returns the expiration seconds expected.
+func GetDefaultExpiration(dsecs string) (time.Duration, error) {
+	defaultExpiryDuration := time.Duration(60) * time.Minute // Defaults to 1hr.
+	if dsecs != "" {
+		expirySecs, err := strconv.ParseInt(dsecs, 10, 64)
+		if err != nil {
+			return 0, auth.ErrInvalidDuration
+		}
+
+		// The duration, in seconds, of the role session.
+		// The value can range from 900 seconds (15 minutes)
+		// up to 365 days.
+		if expirySecs < 900 || expirySecs > 31536000 {
+			return 0, auth.ErrInvalidDuration
+		}
+
+		defaultExpiryDuration = time.Duration(expirySecs) * time.Second
+	}
+	return defaultExpiryDuration, nil
+}

internal/config/identity/openid/providercfg.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package openid
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/minio/minio/internal/arn"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid/provider"
+	xhttp "github.com/minio/minio/internal/http"
+	xnet "github.com/minio/pkg/net"
+)
+
+type providerCfg struct {
+	// Used for user interface like console
+	DisplayName string
+
+	JWKS struct {
+		URL *xnet.URL
+	}
+	URL                *xnet.URL
+	ClaimPrefix        string
+	ClaimName          string
+	ClaimUserinfo      bool
+	RedirectURI        string
+	RedirectURIDynamic bool
+	DiscoveryDoc       DiscoveryDoc
+	ClientID           string
+	ClientSecret       string
+	RolePolicy         string
+
+	roleArn  arn.ARN
+	provider provider.Provider
+}
+
+func newProviderCfgFromConfig(getCfgVal func(env, cfgName string) string) providerCfg {
+	return providerCfg{
+		DisplayName:        getCfgVal(EnvIdentityOpenIDDisplayName, DisplayName),
+		ClaimName:          getCfgVal(EnvIdentityOpenIDClaimName, ClaimName),
+		ClaimUserinfo:      getCfgVal(EnvIdentityOpenIDClaimUserInfo, ClaimUserinfo) == config.EnableOn,
+		ClaimPrefix:        getCfgVal(EnvIdentityOpenIDClaimPrefix, ClaimPrefix),
+		RedirectURI:        getCfgVal(EnvIdentityOpenIDRedirectURI, RedirectURI),
+		RedirectURIDynamic: getCfgVal(EnvIdentityOpenIDRedirectURIDynamic, RedirectURIDynamic) == config.EnableOn,
+		ClientID:           getCfgVal(EnvIdentityOpenIDClientID, ClientID),
+		ClientSecret:       getCfgVal(EnvIdentityOpenIDClientSecret, ClientSecret),
+		RolePolicy:         getCfgVal(EnvIdentityOpenIDRolePolicy, RolePolicy),
+	}
+}
+
+const (
+	keyCloakVendor = "keycloak"
+)
+
+// initializeProvider initializes if any additional vendor specific information
+// was provided, initialization will return an error initial login fails.
+func (p *providerCfg) initializeProvider(cfgGet func(string, string) string, transport http.RoundTripper) error {
+	vendor := cfgGet(EnvIdentityOpenIDVendor, Vendor)
+	if vendor == "" {
+		return nil
+	}
+	var err error
+	switch vendor {
+	case keyCloakVendor:
+		adminURL := cfgGet(EnvIdentityOpenIDKeyCloakAdminURL, KeyCloakAdminURL)
+		realm := cfgGet(EnvIdentityOpenIDKeyCloakRealm, KeyCloakRealm)
+		p.provider, err = provider.KeyCloak(
+			provider.WithAdminURL(adminURL),
+			provider.WithOpenIDConfig(provider.DiscoveryDoc(p.DiscoveryDoc)),
+			provider.WithTransport(transport),
+			provider.WithRealm(realm),
+		)
+		return err
+	default:
+		return fmt.Errorf("Unsupport vendor %s", keyCloakVendor)
+	}
+}
+
+// UserInfo returns claims for authenticated user from userInfo endpoint.
+//
+// Some OIDC implementations such as GitLab do not support
+// claims as part of the normal oauth2 flow, instead rely
+// on service providers making calls to IDP to fetch additional
+// claims available from the UserInfo endpoint
+func (p *providerCfg) UserInfo(accessToken string, transport http.RoundTripper) (map[string]interface{}, error) {
+	if p.JWKS.URL == nil || p.JWKS.URL.String() == "" {
+		return nil, errors.New("openid not configured")
+	}
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, p.DiscoveryDoc.UserInfoEndpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if accessToken != "" {
+		req.Header.Set("Authorization", "Bearer "+accessToken)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		// uncomment this for debugging when needed.
+		// reqBytes, _ := httputil.DumpRequest(req, false)
+		// fmt.Println(string(reqBytes))
+		// respBytes, _ := httputil.DumpResponse(resp, true)
+		// fmt.Println(string(respBytes))
+		return nil, errors.New(resp.Status)
+	}
+
+	dec := json.NewDecoder(resp.Body)
+	claims := map[string]interface{}{}
+
+	if err = dec.Decode(&claims); err != nil {
+		// uncomment this for debugging when needed.
+		// reqBytes, _ := httputil.DumpRequest(req, false)
+		// fmt.Println(string(reqBytes))
+		// respBytes, _ := httputil.DumpResponse(resp, true)
+		// fmt.Println(string(respBytes))
+		return nil, err
+	}
+
+	return claims, nil
+}

internal/config/notify/help.go

@@ -28,9 +28,18 @@ const (
 	queueLimitComment = `maximum limit for undelivered messages, defaults to '100000'`
 )
 
+var enableHelp = config.HelpKV{
+	Key:         config.Enable,
+	Description: "Enable or disable notifications",
+	Type:        "on|off",
+	Sensitive:   false,
+	Optional:    true,
+}
+
 // Help template inputs for all notification targets
 var (
 	HelpWebhook = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.WebhookEndpoint,
 			Description: "webhook server endpoint e.g. http://localhost:8080/minio/events",
@@ -79,6 +88,7 @@ var (
 	}
 
 	HelpAMQP = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.AmqpURL,
 			Description: "AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`",
@@ -167,6 +177,7 @@ var (
 	}
 
 	HelpKafka = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.KafkaBrokers,
 			Description: "comma separated list of Kafka broker addresses",
@@ -263,6 +274,7 @@ var (
 	}
 
 	HelpMQTT = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.MqttBroker,
 			Description: "MQTT server endpoint e.g. `tcp://localhost:1883`",
@@ -327,6 +339,7 @@ var (
 	}
 
 	HelpPostgres = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.PostgresConnectionString,
 			Description: `Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"`,
@@ -370,6 +383,7 @@ var (
 	}
 
 	HelpMySQL = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.MySQLDSNString,
 			Description: `MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>"`,
@@ -414,6 +428,7 @@ var (
 	}
 
 	HelpNATS = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.NATSAddress,
 			Description: "NATS server address e.g. '0.0.0.0:4222'",
@@ -530,6 +545,7 @@ var (
 	}
 
 	HelpNSQ = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.NSQAddress,
 			Description: "NSQ server address e.g. '127.0.0.1:4150'",
@@ -574,6 +590,7 @@ var (
 	}
 
 	HelpES = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.ElasticURL,
 			Description: "Elasticsearch server's address, with optional authentication info",
@@ -625,6 +642,7 @@ var (
 	}
 
 	HelpRedis = config.HelpKVS{
+		enableHelp,
 		config.HelpKV{
 			Key:         target.RedisAddress,
 			Description: "Redis server's address. For example: `localhost:6379`",