JWKS should be parsed before usage (#14842)

fixes #14811

.github/markdown-lint-cfg.yaml

@@ -0,0 +1,5 @@
+# Config file for markdownlint-cli
+MD033:
+  allowed_elements:
+    - details
+    - summary

.github/workflows/go-cross.yml

@@ -11,19 +11,23 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build Tests with Go ${{ matrix.go-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.17.x, 1.18.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+      - uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/go-healing.yml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
@@ -21,9 +24,10 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |
@@ -43,3 +47,4 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify-healing
+          make verify-healing-inconsistent-versions

.github/workflows/go-lint.yml

@@ -11,19 +11,23 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.17.x, 1.18.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         if: matrix.os == 'ubuntu-latest'
         with:

.github/workflows/go.yml

@@ -11,19 +11,23 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.17.x, 1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |

.github/workflows/iam-integrations.yaml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   iam-matrix-test:
     name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]"
@@ -44,6 +47,14 @@ jobs:
           - "5556:5556"
         env:
           DEX_LDAP_SERVER: "openldap:389"
+      openid2:
+        image: quay.io/minio/dex
+        ports:
+          - "5557:5557"
+        env:
+          DEX_LDAP_SERVER: "openldap:389"
+          DEX_ISSUER: "http://127.0.0.1:5557/dex"
+          DEX_WEB_HTTP: "0.0.0.0:5557"
 
     strategy:
       # When ldap, etcd or openid vars are empty below, those external servers
@@ -65,9 +76,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |
@@ -85,6 +97,17 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
+      - name: Test with multiple OpenID providers
+        if: matrix.openid == 'http://127.0.0.1:5556/dex'
+        env:
+          LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          ETCD_SERVER: ${{ matrix.etcd }}
+          OPENID_TEST_SERVER: ${{ matrix.openid }}
+          OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-iam
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |

.github/workflows/markdown-lint.yaml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   lint:
     name: Lint all docs
@@ -22,4 +25,6 @@ jobs:
     - name: Lint all docs
       run: |
         npm install -g markdownlint-cli
-        markdownlint --fix '**/*.md' --disable MD013 MD040
+        markdownlint --fix '**/*.md' \
+          --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
+          --disable MD013 MD040

.github/workflows/replication.yaml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   replication-test:
     name: Replication Tests with Go ${{ matrix.go-version }}
@@ -22,9 +25,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |

.github/workflows/upgrade-ci-cd.yaml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
@@ -22,10 +25,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
-
+          check-latest: true
       - name: Start upgrade tests
         run: |
           make test-upgrade

.golangci.yml

@@ -25,6 +25,8 @@ linters:
     - varcheck
     - gocritic
     - gofumpt
+    - tenv
+    - durationcheck
 
 linters-settings:
   gofumpt:

Makefile

@@ -19,7 +19,7 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2
 	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
@@ -82,6 +82,11 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
+verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
+	@echo "Verify resolving inconsistent versions build with race"
+	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
+
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

buildscripts/resolve-right-versions.sh

@@ -0,0 +1,72 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+function start_minio_5drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export MINIO_CI_CD=1
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" stat minio/bucket/testobj
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_5drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

cmd/admin-bucket-handlers.go

@@ -120,7 +120,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	config, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
+	config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers-config-kv.go

@@ -97,7 +97,8 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 }
 
 func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config, subSys string,
-	r *http.Request, w http.ResponseWriter) {
+	r *http.Request, w http.ResponseWriter,
+) {
 	// Apply dynamic values.
 	if err := applyDynamicConfigForSubSys(GlobalContext, objectAPI, cfg, subSys); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)

cmd/admin-handlers-pools.go

@@ -58,6 +58,16 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
+		for nodeIdx, proxyEp := range globalProxyEndpoints {
+			if proxyEp.Endpoint.Host == ep.Host {
+				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+					return
+				}
+			}
+		}
+	}
+
 	if err := pools.Decommission(r.Context(), idx); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -96,6 +106,16 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
+		for nodeIdx, proxyEp := range globalProxyEndpoints {
+			if proxyEp.Endpoint.Host == ep.Host {
+				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+					return
+				}
+			}
+		}
+	}
+
 	if err := pools.DecommissionCancel(ctx, idx); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers-site-replication.go

@@ -115,10 +115,12 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 	case madmin.MakeWithVersioningBktOp:
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
+		_, isForceCreate := r.Form["forceCreate"]
 		opts := BucketOptions{
 			Location:          r.Form.Get("location"),
 			LockEnabled:       isLockEnabled,
 			VersioningEnabled: isVersioningEnabled,
+			ForceCreate:       isForceCreate,
 		}
 		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 	case madmin.ConfigureReplBktOp:

cmd/admin-handlers-users-race_test.go

@@ -73,7 +73,7 @@ func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) {
 }
 
 func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
 	defer cancel()
 
 	bucket := getRandomBucketName()

cmd/admin-handlers-users.go

@@ -90,7 +90,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
 
 	password := cred.SecretKey
 
-	allCredentials, err := globalIAMSys.ListBucketUsers(bucket)
+	allCredentials, err := globalIAMSys.ListBucketUsers(ctx, bucket)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -124,12 +124,24 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 
 	password := cred.SecretKey
 
-	allCredentials, err := globalIAMSys.ListUsers()
+	allCredentials, err := globalIAMSys.ListUsers(ctx)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	// Add ldap users which have mapped policies if in LDAP mode
+	// FIXME(vadmeste): move this to policy info in the future
+	ldapUsers, err := globalIAMSys.ListLDAPUsers()
+	if err != nil && err != errIAMActionNotAllowed {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	for k, v := range ldapUsers {
+		allCredentials[k] = v
+	}
+
+	// Marshal the response
 	data, err := json.Marshal(allCredentials)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -605,7 +617,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
 			Claims:          claims,
-			DenyOnly:        true,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
@@ -1171,8 +1182,8 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 
 			lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
 			quota, _ := globalBucketQuotaSys.Get(ctx, bucket.Name)
-			rcfg, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name)
-			tcfg, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
+			rcfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name)
+			tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
 
 			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
 				Name:                 bucket.Name,

cmd/admin-handlers-users_test.go

@@ -50,6 +50,8 @@ const (
 type TestSuiteIAM struct {
 	TestSuiteCommon
 
+	ServerTypeDescription string
+
 	// Flag to turn on tests for etcd backend IAM
 	withEtcdBackend bool
 
@@ -59,7 +61,15 @@ type TestSuiteIAM struct {
 }
 
 func newTestSuiteIAM(c TestSuiteCommon, withEtcdBackend bool) *TestSuiteIAM {
-	return &TestSuiteIAM{TestSuiteCommon: c, withEtcdBackend: withEtcdBackend}
+	etcdStr := ""
+	if withEtcdBackend {
+		etcdStr = " (with etcd backend)"
+	}
+	return &TestSuiteIAM{
+		TestSuiteCommon:       c,
+		ServerTypeDescription: fmt.Sprintf("%s%s", c.serverType, etcdStr),
+		withEtcdBackend:       withEtcdBackend,
+	}
 }
 
 func (s *TestSuiteIAM) iamSetup(c *check) {
@@ -890,6 +900,9 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 
 	// 5. Check that service account can be deleted.
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
+
+	// 6. Check that service account cannot be created for some other user.
+	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
 }
 
 func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {

cmd/admin-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,15 +18,16 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/subtle"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -500,7 +501,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 }
 
 // StartProfilingResult contains the status of the starting
-// profiling action in a given server
+// profiling action in a given server - deprecated API
 type StartProfilingResult struct {
 	NodeName string `json:"nodeName"`
 	Success  bool   `json:"success"`
@@ -594,6 +595,85 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	writeSuccessResponseJSON(w, startProfilingResultInBytes)
 }
 
+// ProfileHandler - POST /minio/admin/v3/profile/?profilerType={profilerType}
+// ----------
+// Enable server profiling
+func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "Profile")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	profileStr := r.Form.Get("profilerType")
+	profiles := strings.Split(profileStr, ",")
+	duration := time.Minute
+	if dstr := r.Form.Get("duration"); dstr != "" {
+		var err error
+		duration, err = time.ParseDuration(dstr)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+			return
+		}
+	}
+	// read request body
+	io.CopyN(ioutil.Discard, r.Body, 1)
+
+	globalProfilerMu.Lock()
+
+	if globalProfiler == nil {
+		globalProfiler = make(map[string]minioProfiler, 10)
+	}
+
+	// Stop profiler of all types if already running
+	for k, v := range globalProfiler {
+		v.Stop()
+		delete(globalProfiler, k)
+	}
+
+	// Start profiling on remote servers.
+	for _, profiler := range profiles {
+		globalNotificationSys.StartProfiling(profiler)
+
+		// Start profiling locally as well.
+		prof, err := startProfiler(profiler)
+		if err == nil {
+			globalProfiler[profiler] = prof
+		}
+	}
+	globalProfilerMu.Unlock()
+
+	timer := time.NewTimer(duration)
+	for {
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			for k, v := range globalProfiler {
+				v.Stop()
+				delete(globalProfiler, k)
+			}
+			return
+		case <-timer.C:
+			if !globalNotificationSys.DownloadProfilingData(ctx, w) {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminProfilerNotEnabled), r.URL)
+				return
+			}
+			return
+		}
+	}
+}
+
 // dummyFileInfo represents a dummy representation of a profile data file
 // present only in memory, it helps to generate the zip stream.
 type dummyFileInfo struct {
@@ -614,7 +694,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 
 // DownloadProfilingHandler - POST /minio/admin/v3/profiling/download
 // ----------
-// Download profiling information of all nodes in a zip format
+// Download profiling information of all nodes in a zip format - deprecated API
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
@@ -1029,38 +1109,30 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		duration = time.Second * 10
 	}
 
-	// ignores any errors here.
-	storageInfo, _ := objectAPI.StorageInfo(ctx)
-	capacityNeeded := uint64(concurrent * size)
-	capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
+	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
 
-	if capacity < capacityNeeded {
+	if !sufficientCapacity {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
-			Code: "XMinioSpeedtestInsufficientCapacity",
-			Message: fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
-				humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)),
+			Code:       "XMinioSpeedtestInsufficientCapacity",
+			Message:    capacityErrMsg,
 			StatusCode: http.StatusInsufficientStorage,
 		}), r.URL)
 		return
 	}
 
-	// Verify if we can employ autotune without running out of capacity,
-	// if we do run out of capacity, make sure to turn-off autotuning
-	// in such situations.
-	newConcurrent := concurrent + (concurrent+1)/2
-	autoTunedCapacityNeeded := uint64(newConcurrent * size)
-	if autotune && capacity < autoTunedCapacityNeeded {
-		// Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity.
+	if autotune && !canAutotune {
 		autotune = false
 	}
 
-	deleteBucket := func() {
-		objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaBucket, "speedtest"), DeleteBucketOptions{
-			Force:      true,
-			NoRecreate: true,
-		})
+	bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if !bucketExists {
+		defer deleteObjectPerfBucket(objectAPI)
 	}
-	defer deleteBucket()
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1072,7 +1144,7 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
-	ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
+	ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
 	for {
 		select {
 		case <-ctx.Done():
@@ -1095,6 +1167,50 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	}
 }
 
+func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) {
+	err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
+	if err != nil {
+		if _, ok := err.(BucketExists); !ok {
+			// Only BucketExists error can be ignored.
+			return false, err
+		}
+		bucketExists = true
+	}
+	return bucketExists, nil
+}
+
+func deleteObjectPerfBucket(objectAPI ObjectLayer) {
+	objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
+		Force:      true,
+		NoRecreate: true,
+	})
+}
+
+func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
+	capacityNeeded := uint64(concurrent * size)
+	capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
+
+	if capacity < capacityNeeded {
+		return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
+			humanize.IBytes(capacityNeeded), humanize.IBytes(capacity))
+	}
+
+	// Verify if we can employ autotune without running out of capacity,
+	// if we do run out of capacity, make sure to turn-off autotuning
+	// in such situations.
+	if autotune {
+		newConcurrent := concurrent + (concurrent+1)/2
+		autoTunedCapacityNeeded := uint64(newConcurrent * size)
+		if capacity < autoTunedCapacityNeeded {
+			// Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity.
+			return true, false, ""
+		}
+	}
+
+	return true, autotune, ""
+}
+
 // NetSpeedtestHandler - reports maximum network throughput
 func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NetSpeedtestHandler")
@@ -1583,6 +1699,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 	buckets := madmin.Buckets{}
 	objects := madmin.Objects{}
+	versions := madmin.Versions{}
 	usage := madmin.Usage{}
 
 	objectAPI := newObjectLayerFn()
@@ -1594,6 +1711,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		if err == nil {
 			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
 			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
+			versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount}
 			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
 		} else {
 			buckets = madmin.Buckets{Error: err.Error()}
@@ -1642,6 +1760,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
 		Objects:      objects,
+		Versions:     versions,
 		Usage:        usage,
 		Services:     services,
 		Backend:      backend,
@@ -1663,7 +1782,14 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	query := r.Form
-	healthInfo := madmin.HealthInfo{Version: madmin.HealthInfoVersion}
+	healthInfo := madmin.HealthInfo{
+		Version: madmin.HealthInfoVersion,
+		Minio: madmin.MinioHealthInfo{
+			Info: madmin.MinioInfo{
+				DeploymentID: globalDeploymentID,
+			},
+		},
+	}
 	healthInfoCh := make(chan madmin.HealthInfo)
 
 	enc := json.NewEncoder(w)
@@ -1685,7 +1811,12 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		logger.LogIf(ctx, enc.Encode(healthInfo))
 	}
 
-	deadline := 1 * time.Hour
+	deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
+	if query.Get(string(madmin.HealthDataTypePerfNet)) != "" ||
+		query.Get(string(madmin.HealthDataTypePerfDrive)) != "" ||
+		query.Get(string(madmin.HealthDataTypePerfObj)) != "" {
+		deadline = 1 * time.Hour
+	}
 	if dstr := r.Form.Get("deadline"); dstr != "" {
 		var err error
 		deadline, err = time.ParseDuration(dstr)
@@ -1928,49 +2059,98 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	getAndWriteDrivePerfInfo := func() {
-		if query.Get("perfdrive") == "true" {
-			localDPI := getDrivePerfInfos(deadlinedCtx, globalLocalNodeName)
-			anonymizeAddr(&localDPI)
-			healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, localDPI)
-			partialWrite(healthInfo)
+		if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" {
+			// Freeze all incoming S3 API calls before running speedtest.
+			globalNotificationSys.ServiceFreeze(ctx, true)
+			// unfreeze all incoming S3 API calls after speedtest.
+			defer globalNotificationSys.ServiceFreeze(ctx, false)
 
-			perfCh := globalNotificationSys.GetDrivePerfInfos(deadlinedCtx)
-			for perfInfo := range perfCh {
-				anonymizeAddr(&perfInfo)
-				healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, perfInfo)
-				partialWrite(healthInfo)
+			opts := madmin.DriveSpeedTestOpts{
+				Serial:    false,
+				BlockSize: 4 * humanize.MiByte,
+				FileSize:  1 * humanize.GiByte,
 			}
+
+			localDPI := driveSpeedTest(ctx, opts)
+			healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI)
+
+			perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts)
+			for perfInfo := range perfCh {
+				healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo)
+			}
+			partialWrite(healthInfo)
 		}
 	}
 
-	anonymizeNetPerfInfo := func(npi *madmin.NetPerfInfo) {
-		anonymizeAddr(npi)
-		rps := npi.RemotePeers
-		for idx, peer := range rps {
-			anonymizeAddr(&peer)
-			rps[idx] = peer
+	getAndWriteObjPerfInfo := func() {
+		if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
+			concurrent := 32
+			size := 64 * humanize.MiByte
+			autotune := true
+
+			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
+
+			if !sufficientCapacity {
+				healthInfo.Perf.Error = capacityErrMsg
+				partialWrite(healthInfo)
+				return
+			}
+
+			if !canAutotune {
+				autotune = false
+			}
+
+			bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
+			if err != nil {
+				healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error()
+				partialWrite(healthInfo)
+				return
+			}
+
+			if !bucketExists {
+				defer deleteObjectPerfBucket(objectAPI)
+			}
+
+			// Freeze all incoming S3 API calls before running speedtest.
+			globalNotificationSys.ServiceFreeze(ctx, true)
+
+			// unfreeze all incoming S3 API calls after speedtest.
+			defer globalNotificationSys.ServiceFreeze(ctx, false)
+
+			opts := speedTestOpts{
+				throughputSize:   size,
+				concurrencyStart: concurrent,
+				duration:         10 * time.Second,
+				autotune:         autotune,
+			}
+
+			perfCh := objectSpeedTest(ctx, opts)
+			for perfInfo := range perfCh {
+				healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo)
+			}
+			partialWrite(healthInfo)
 		}
-		npi.RemotePeers = rps
 	}
 
 	getAndWriteNetPerfInfo := func() {
-		if globalIsDistErasure && query.Get("perfnet") == "true" {
-			localNPI := globalNotificationSys.GetNetPerfInfo(deadlinedCtx)
-			anonymizeNetPerfInfo(&localNPI)
-			healthInfo.Perf.Net = append(healthInfo.Perf.Net, localNPI)
-
-			partialWrite(healthInfo)
-
-			netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
-			for netInfo := range netInfos {
-				anonymizeNetPerfInfo(&netInfo)
-				healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo)
-				partialWrite(healthInfo)
+		if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
+			if !globalIsDistErasure {
+				return
 			}
 
-			ppi := globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx)
-			anonymizeNetPerfInfo(&ppi)
-			healthInfo.Perf.NetParallel = ppi
+			nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
+			lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
+				healthInfo.Perf.Error = "Could not acquire lock for netperf: " + err.Error()
+			} else {
+				defer nsLock.Unlock(lkctx.Cancel)
+
+				netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
+				for _, np := range netPerf {
+					np.Endpoint = anonAddr(np.Endpoint)
+					healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
+				}
+			}
 			partialWrite(healthInfo)
 		}
 	}
@@ -1996,7 +2176,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	go func() {
 		defer close(healthInfoCh)
 
-		partialWrite(healthInfo) // Write first message with only version populated
+		partialWrite(healthInfo) // Write first message with only version and deployment id populated
 		getAndWriteCPUs()
 		getAndWritePartitions()
 		getAndWriteOSInfo()
@@ -2004,6 +2184,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		getAndWriteProcInfo()
 		getAndWriteMinioConfig()
 		getAndWriteDrivePerfInfo()
+		getAndWriteObjPerfInfo()
 		getAndWriteNetPerfInfo()
 		getAndWriteSysErrors()
 		getAndWriteSysServices()
@@ -2332,19 +2513,9 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 	defer cancel()
 
-	client := &http.Client{Transport: &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(timeout),
-		ResponseHeaderTimeout: 5 * time.Second,
-		TLSHandshakeTimeout:   5 * time.Second,
-		ExpectContinueTimeout: 5 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
-		// Go net/http automatically unzip if content-type is
-		// gzip disable this feature, as we are always interested
-		// in raw stream.
-		DisableCompression: true,
-	}}
-	defer client.CloseIdleConnections()
+	client := &http.Client{
+		Transport: globalProxyTransport,
+	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
 	if err != nil {
@@ -2437,8 +2608,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	// of profiling data of all nodes
 	zipWriter := zip.NewWriter(encw)
 	defer zipWriter.Close()
-
-	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, si StatInfo) error {
+	rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error {
 		// Prefix host+disk
 		filename = path.Join(host, disk, filename)
 		if si.Dir {
@@ -2449,6 +2619,10 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			// Not, set it to default.
 			si.Mode = 0o600
 		}
+		if si.ModTime.IsZero() {
+			// Set time to now.
+			si.ModTime = time.Now()
+		}
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
 			size:    si.Size,
@@ -2471,10 +2645,33 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			logger.LogIf(ctx, err)
 		}
 		return nil
-	})
+	}
+	err = o.GetRawData(ctx, volume, file, rawDataFn)
 	if !errors.Is(err, errFileNotFound) {
 		logger.LogIf(ctx, err)
 	}
+
+	// save the format.json as part of inspect by default
+	if volume != minioMetaBucket && file != formatConfigFile {
+		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
+	}
+	if !errors.Is(err, errFileNotFound) {
+		logger.LogIf(ctx, err)
+	}
+	// save args passed to inspect command
+	inspectArgs := []string{fmt.Sprintf(" Inspect path: %s%s%s\n", volume, slashSeparator, file)}
+	cmdLine := []string{"Server command line args: "}
+	for _, pool := range globalEndpoints {
+		cmdLine = append(cmdLine, pool.CmdLine)
+	}
+	cmdLine = append(cmdLine, "\n")
+	inspectArgs = append(inspectArgs, cmdLine...)
+	inspectArgsBytes := []byte(strings.Join(inspectArgs, " "))
+	if err = rawDataFn(bytes.NewReader(inspectArgsBytes), "", "", "inspect-input.txt", StatInfo{
+		Size: int64(len(inspectArgsBytes)),
+	}); err != nil {
+		logger.LogIf(ctx, err)
+	}
 }
 
 func createHostAnonymizerForFSMode() map[string]string {
@@ -2520,6 +2717,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
 	if !found {
 		// In distributed setup, anonymized addr = 'poolNum.serverNum'
 		newHost := fmt.Sprintf("pool%d.server%d", poolNum, srvrNum)
+		schemePfx := endpoint.Scheme + "://"
 
 		// Hostname
 		mapIfNotPresent(hostAnonymizer, endpoint.Hostname(), newHost)
@@ -2529,6 +2727,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
 			// Host + port
 			newHostPort = newHost + ":" + endpoint.Port()
 			mapIfNotPresent(hostAnonymizer, endpoint.Host, newHostPort)
+			mapIfNotPresent(hostAnonymizer, schemePfx+endpoint.Host, newHostPort)
 		}
 
 		newHostPortPath := newHostPort
@@ -2537,10 +2736,11 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
 			currentHostPortPath := endpoint.Host + endpoint.Path
 			newHostPortPath = newHostPort + endpoint.Path
 			mapIfNotPresent(hostAnonymizer, currentHostPortPath, newHostPortPath)
+			mapIfNotPresent(hostAnonymizer, schemePfx+currentHostPortPath, newHostPortPath)
 		}
 
 		// Full url
-		hostAnonymizer[currentURL] = endpoint.Scheme + "://" + newHostPortPath
+		hostAnonymizer[currentURL] = schemePfx + newHostPortPath
 	}
 }
 

cmd/admin-heal-ops.go

@@ -701,7 +701,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	if source.opts != nil {
 		task.opts = *source.opts
 	} else {
-		task.opts.ScanMode = globalHealConfig.ScanMode()
+		task.opts.ScanMode = madmin.HealNormalScan
 	}
 
 	h.mutex.Lock()

cmd/admin-router.go

@@ -84,10 +84,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
 		}
 
-		// Profiling operations
+		// Profiling operations - deprecated API
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(gz(httpTraceAll(adminAPI.StartProfilingHandler))).
 			Queries("profilerType", "{profilerType:.*}")
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(gz(httpTraceAll(adminAPI.DownloadProfilingHandler)))
+		// Profiling operations
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(gz(httpTraceAll(adminAPI.ProfileHandler)))
 
 		// Config KV operations.
 		if enableConfigOps {

cmd/admin-server-info.go

@@ -50,7 +50,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := isServerResolvable(endpoint, 2*time.Second); err == nil {
+				if err := isServerResolvable(endpoint, 5*time.Second); err == nil {
 					network[nodeName] = string(madmin.ItemOnline)
 				} else {
 					network[nodeName] = string(madmin.ItemOffline)

cmd/auth-handler.go

@@ -197,13 +197,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }
 
-// Fetch claims in the security token returned by the client.
-func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	if token == "" {
-		claims := xjwt.NewMapClaims()
-		return claims.Map(), nil
-	}
-
+func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -212,9 +206,15 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	// hijacking the policies. We need to make sure that this is
 	// based an admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
-	claims, err := auth.ExtractClaims(token, globalActiveCred.SecretKey)
+	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
-		return nil, errAuthentication
+		if subtle.ConstantTimeCompare([]byte(secret), []byte(globalActiveCred.SecretKey)) == 1 {
+			return nil, errAuthentication
+		}
+		claims, err = auth.ExtractClaims(token, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, errAuthentication
+		}
 	}
 
 	// If OPA is set, return without any further checks.
@@ -241,23 +241,50 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	return claims.Map(), nil
 }
 
+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(token string) (map[string]interface{}, error) {
+	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+}
+
 // Fetch claims in the security token returned by the client and validate the token.
 func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
 	token := getSessionToken(r)
 	if token != "" && cred.AccessKey == "" {
+		// x-amz-security-token is not allowed for anonymous access.
 		return nil, ErrNoAccessKey
 	}
-	if cred.IsServiceAccount() && token == "" {
-		token = cred.SessionToken
-	}
-	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+
+	if token == "" && cred.IsTemp() {
+		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
-	claims, err := getClaimsFromToken(token)
-	if err != nil {
-		return nil, toAPIErrorCode(r.Context(), err)
+
+	if token != "" && !cred.IsTemp() {
+		// x-amz-security-token should not present for static credentials.
+		return nil, ErrInvalidToken
 	}
-	return claims, ErrNone
+
+	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		// validate token for temporary credentials only.
+		return nil, ErrInvalidToken
+	}
+
+	secret := globalActiveCred.SecretKey
+	if cred.IsServiceAccount() {
+		token = cred.SessionToken
+		secret = cred.SecretKey
+	}
+
+	if token != "" {
+		claims, err := getClaimsFromTokenWithSecret(token, secret)
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		return claims, ErrNone
+	}
+
+	claims := xjwt.NewMapClaims()
+	return claims.Map(), ErrNone
 }
 
 // Check request auth type verifies the incoming http request

cmd/auth-handler_test.go

@@ -341,7 +341,8 @@ func mustNewSignedEmptyMD5Request(method string, urlStr string, contentLength in
 }
 
 func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64,
-	body io.ReadSeeker, t *testing.T) *http.Request {
+	body io.ReadSeeker, t *testing.T,
+) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
 	cred := globalActiveCred

cmd/bucket-encryption-handlers.go

@@ -158,7 +158,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	config, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+	config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-encryption.go

@@ -43,7 +43,8 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error)
 		return nil, BucketSSEConfigNotFound{Bucket: bucket}
 	}
 
-	return globalBucketMetadataSys.GetSSEConfig(bucket)
+	sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+	return sseCfg, err
 }
 
 // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.

cmd/bucket-handlers.go

@@ -706,13 +706,27 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 
 	objectLockEnabled := false
-	if vs, found := r.Header[http.CanonicalHeaderKey("x-amz-bucket-object-lock-enabled")]; found {
-		v := strings.ToLower(strings.Join(vs, ""))
-		if v != "true" && v != "false" {
+	if vs := r.Header.Get(xhttp.AmzObjectLockEnabled); len(vs) > 0 {
+		v := strings.ToLower(vs)
+		switch v {
+		case "true", "false":
+			objectLockEnabled = v == "true"
+		default:
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+			return
+		}
+	}
+
+	forceCreate := false
+	if vs := r.Header.Get(xhttp.MinIOForceCreate); len(vs) > 0 {
+		v := strings.ToLower(vs)
+		switch v {
+		case "true", "false":
+			forceCreate = v == "true"
+		default:
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 			return
 		}
-		objectLockEnabled = v == "true"
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
@@ -737,6 +751,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	opts := BucketOptions{
 		Location:    location,
 		LockEnabled: objectLockEnabled,
+		ForceCreate: forceCreate,
 	}
 
 	if globalDNSConfig != nil {
@@ -1374,7 +1389,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	}
 
 	// Deny object locking configuration settings on existing buckets without object lock enabled.
-	if _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
+	if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1427,7 +1442,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	config, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+	config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1529,7 +1544,7 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	config, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
+	config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1677,7 +1692,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 
-	config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1820,7 +1835,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		return
 	}
 
-	config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1908,7 +1923,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 		return
 	}
 
-	if _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-handlers_test.go

@@ -36,7 +36,8 @@ func TestRemoveBucketHandler(t *testing.T) {
 }
 
 func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials auth.Credentials, t *testing.T,
+) {
 	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewReader([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
 	// if object upload fails stop the test.
 	if err != nil {

cmd/bucket-lifecycle-handlers_test.go

@@ -34,7 +34,8 @@ func TestBucketLifecycleWrongCredentials(t *testing.T) {
 
 // Test for authentication
 func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials auth.Credentials, t *testing.T,
+) {
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		method     string

cmd/bucket-lifecycle.go

@@ -247,11 +247,11 @@ func (t *transitionState) addLastDayStats(tier string, ts tierStats) {
 	t.lastDayStats[tier].addStats(ts)
 }
 
-func (t *transitionState) getDailyAllTierStats() dailyAllTierStats {
+func (t *transitionState) getDailyAllTierStats() DailyAllTierStats {
 	t.lastDayMu.RLock()
 	defer t.lastDayMu.RUnlock()
 
-	res := make(dailyAllTierStats, len(t.lastDayStats))
+	res := make(DailyAllTierStats, len(t.lastDayStats))
 	for tier, st := range t.lastDayStats {
 		res[tier] = st.clone()
 	}

cmd/bucket-listobjects-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -26,26 +27,9 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/logger"
 
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
 )
 
-func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
-	g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
-	for index := range objects {
-		index := index
-		g.Go(func() error {
-			size, err := objects[index].GetActualSize()
-			if err == nil {
-				objects[index].Size = size
-			}
-			objects[index].ETag = objects[index].GetActualETag(nil)
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
@@ -116,7 +100,11 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectVersionsInfo.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
@@ -178,7 +166,11 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
@@ -253,7 +245,11 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -350,7 +346,11 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectsInfo.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectsInfo.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 

cmd/bucket-metadata-sys.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -111,21 +112,26 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 	switch configFile {
 	case bucketPolicyConfig:
 		meta.PolicyConfigJSON = configData
+		meta.PolicyConfigUpdatedAt = UTCNow()
 	case bucketNotificationConfig:
 		meta.NotificationConfigXML = configData
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
+		meta.EncryptionConfigUpdatedAt = UTCNow()
 	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
+		meta.TaggingConfigUpdatedAt = UTCNow()
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
+		meta.QuotaConfigUpdatedAt = UTCNow()
 	case objectLockConfig:
 		if !globalIsErasure && !globalIsDistErasure {
 			return NotImplemented{}
 		}
 		meta.ObjectLockConfigXML = configData
+		meta.ObjectLockConfigUpdatedAt = UTCNow()
 	case bucketVersioningConfig:
 		if !globalIsErasure && !globalIsDistErasure {
 			return NotImplemented{}
@@ -136,6 +142,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 			return NotImplemented{}
 		}
 		meta.ReplicationConfigXML = configData
+		meta.ReplicationConfigUpdatedAt = UTCNow()
 	case bucketTargetsFile:
 		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{
 			bucket:            meta.Name,
@@ -196,34 +203,34 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
+func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketTaggingNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.taggingConfig == nil {
-		return nil, BucketTaggingNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
 	}
-	return meta.taggingConfig, nil
+	return meta.taggingConfig, meta.TaggingConfigUpdatedAt, nil
 }
 
 // GetObjectLockConfig returns configured object lock config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) {
+func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, time.Time, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.objectLockConfig == nil {
-		return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
 	}
-	return meta.objectLockConfig, nil
+	return meta.objectLockConfig, meta.ObjectLockConfigUpdatedAt, nil
 }
 
 // GetLifecycleConfig returns configured lifecycle config
@@ -283,18 +290,27 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
 
 // GetSSEConfig returns configured SSE config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) {
+func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, time.Time, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketSSEConfigNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.sseConfig == nil {
-		return nil, BucketSSEConfigNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
 	}
-	return meta.sseConfig, nil
+	return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
+}
+
+// CreatedAt returns the time of creation of bucket
+func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
+	meta, err := sys.GetConfig(GlobalContext, bucket)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return meta.Created.UTC(), nil
 }
 
 // GetPolicyConfig returns configured bucket policy
@@ -323,29 +339,29 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, er
 
 // GetQuotaConfig returns configured bucket quota
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, error) {
+func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
 	meta, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
-	return meta.quotaConfig, nil
+	return meta.quotaConfig, meta.QuotaConfigUpdatedAt, nil
 }
 
 // GetReplicationConfig returns configured bucket replication config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, error) {
+func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, time.Time, error) {
 	meta, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketReplicationConfigNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 
 	if meta.replicationConfig == nil {
-		return nil, BucketReplicationConfigNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
 	}
-	return meta.replicationConfig, nil
+	return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil
 }
 
 // GetBucketTargetsConfig returns configured bucket targets for this bucket

cmd/bucket-metadata.go

@@ -81,6 +81,12 @@ type BucketMetadata struct {
 	ReplicationConfigXML        []byte
 	BucketTargetsConfigJSON     []byte
 	BucketTargetsConfigMetaJSON []byte
+	PolicyConfigUpdatedAt       time.Time
+	ObjectLockConfigUpdatedAt   time.Time
+	EncryptionConfigUpdatedAt   time.Time
+	TaggingConfigUpdatedAt      time.Time
+	QuotaConfigUpdatedAt        time.Time
+	ReplicationConfigUpdatedAt  time.Time
 
 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.Policy
@@ -98,9 +104,10 @@ type BucketMetadata struct {
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
 func newBucketMetadata(name string) BucketMetadata {
+	now := UTCNow()
 	return BucketMetadata{
 		Name:    name,
-		Created: UTCNow(),
+		Created: now,
 		notificationConfig: &event.Config{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
@@ -157,8 +164,13 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
 		return b, err
 	}
+
 	// migrate unencrypted remote targets
-	return b, b.migrateTargetConfig(ctx, objectAPI)
+	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
+		return b, err
+	}
+	b.defaultTimestamps()
+	return b, nil
 }
 
 // parseAllConfigs will parse all configs and populate the private fields.
@@ -347,6 +359,32 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 	return nil
 }
 
+// default timestamps to metadata Created timestamp if unset.
+func (b *BucketMetadata) defaultTimestamps() {
+	if b.PolicyConfigUpdatedAt.IsZero() {
+		b.PolicyConfigUpdatedAt = b.Created
+	}
+	if b.EncryptionConfigUpdatedAt.IsZero() {
+		b.EncryptionConfigUpdatedAt = b.Created
+	}
+
+	if b.TaggingConfigUpdatedAt.IsZero() {
+		b.TaggingConfigUpdatedAt = b.Created
+	}
+
+	if b.ObjectLockConfigUpdatedAt.IsZero() {
+		b.ObjectLockConfigUpdatedAt = b.Created
+	}
+
+	if b.QuotaConfigUpdatedAt.IsZero() {
+		b.QuotaConfigUpdatedAt = b.Created
+	}
+
+	if b.ReplicationConfigUpdatedAt.IsZero() {
+		b.ReplicationConfigUpdatedAt = b.Created
+	}
+}
+
 // Save config to supplied ObjectLayer api.
 func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 	if err := b.parseAllConfigs(ctx, api); err != nil {

cmd/bucket-metadata_gen.go

@@ -108,6 +108,42 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
 				return
 			}
+		case "PolicyConfigUpdatedAt":
+			z.PolicyConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "PolicyConfigUpdatedAt")
+				return
+			}
+		case "ObjectLockConfigUpdatedAt":
+			z.ObjectLockConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt")
+				return
+			}
+		case "EncryptionConfigUpdatedAt":
+			z.EncryptionConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "EncryptionConfigUpdatedAt")
+				return
+			}
+		case "TaggingConfigUpdatedAt":
+			z.TaggingConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "TaggingConfigUpdatedAt")
+				return
+			}
+		case "QuotaConfigUpdatedAt":
+			z.QuotaConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "QuotaConfigUpdatedAt")
+				return
+			}
+		case "ReplicationConfigUpdatedAt":
+			z.ReplicationConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -121,9 +157,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 14
+	// map header, size 20
 	// write "Name"
-	err = en.Append(0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -262,15 +298,75 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
 		return
 	}
+	// write "PolicyConfigUpdatedAt"
+	err = en.Append(0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.PolicyConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "PolicyConfigUpdatedAt")
+		return
+	}
+	// write "ObjectLockConfigUpdatedAt"
+	err = en.Append(0xb9, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.ObjectLockConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt")
+		return
+	}
+	// write "EncryptionConfigUpdatedAt"
+	err = en.Append(0xb9, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.EncryptionConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "EncryptionConfigUpdatedAt")
+		return
+	}
+	// write "TaggingConfigUpdatedAt"
+	err = en.Append(0xb6, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.TaggingConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "TaggingConfigUpdatedAt")
+		return
+	}
+	// write "QuotaConfigUpdatedAt"
+	err = en.Append(0xb4, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.QuotaConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "QuotaConfigUpdatedAt")
+		return
+	}
+	// write "ReplicationConfigUpdatedAt"
+	err = en.Append(0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.ReplicationConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 14
+	// map header, size 20
 	// string "Name"
-	o = append(o, 0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -311,6 +407,24 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "BucketTargetsConfigMetaJSON"
 	o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
 	o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
+	// string "PolicyConfigUpdatedAt"
+	o = append(o, 0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.PolicyConfigUpdatedAt)
+	// string "ObjectLockConfigUpdatedAt"
+	o = append(o, 0xb9, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.ObjectLockConfigUpdatedAt)
+	// string "EncryptionConfigUpdatedAt"
+	o = append(o, 0xb9, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.EncryptionConfigUpdatedAt)
+	// string "TaggingConfigUpdatedAt"
+	o = append(o, 0xb6, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.TaggingConfigUpdatedAt)
+	// string "QuotaConfigUpdatedAt"
+	o = append(o, 0xb4, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.QuotaConfigUpdatedAt)
+	// string "ReplicationConfigUpdatedAt"
+	o = append(o, 0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.ReplicationConfigUpdatedAt)
 	return
 }
 
@@ -416,6 +530,42 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
 				return
 			}
+		case "PolicyConfigUpdatedAt":
+			z.PolicyConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PolicyConfigUpdatedAt")
+				return
+			}
+		case "ObjectLockConfigUpdatedAt":
+			z.ObjectLockConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt")
+				return
+			}
+		case "EncryptionConfigUpdatedAt":
+			z.EncryptionConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "EncryptionConfigUpdatedAt")
+				return
+			}
+		case "TaggingConfigUpdatedAt":
+			z.TaggingConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "TaggingConfigUpdatedAt")
+				return
+			}
+		case "QuotaConfigUpdatedAt":
+			z.QuotaConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "QuotaConfigUpdatedAt")
+				return
+			}
+		case "ReplicationConfigUpdatedAt":
+			z.ReplicationConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -430,6 +580,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON)
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize
 	return
 }

cmd/bucket-object-lock.go

@@ -44,7 +44,7 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 		return r, nil
 	}
 
-	config, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
+	config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
 	if err != nil {
 		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
 			return r, nil

cmd/bucket-policy-handlers_test.go

@@ -113,7 +113,8 @@ func TestCreateBucket(t *testing.T) {
 
 // testCreateBucket - Test for calling Create Bucket and ensure we get one and only one success.
 func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials auth.Credentials, t *testing.T,
+) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
 
 	const n = 100
@@ -378,7 +379,8 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 
 // testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials auth.Credentials, t *testing.T,
+) {
 	// template for constructing HTTP request body for PUT bucket policy.
 	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}`
 

cmd/bucket-quota.go

@@ -42,8 +42,8 @@ func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.
 		}
 		return &madmin.BucketQuota{}, nil
 	}
-
-	return globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
+	qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
+	return qCfg, err
 }
 
 // NewBucketQuotaSys returns initialized BucketQuotaSys

cmd/bucket-replication.go

@@ -81,7 +81,8 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 		return rc, BucketReplicationConfigNotFound{Bucket: bucketName}
 	}
 
-	return globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
+	rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
+	return rCfg, err
 }
 
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
@@ -425,7 +426,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		wg.Add(1)
 		go func(index int, tgt *TargetClient) {
 			defer wg.Done()
-			rinfo := replicateDeleteToTarget(ctx, dobj, objectAPI, tgt)
+			rinfo := replicateDeleteToTarget(ctx, dobj, tgt)
 			rinfos.Targets[index] = rinfo
 		}(idx, tgt)
 	}
@@ -482,7 +483,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	}
 }
 
-func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID

cmd/common-main.go

@@ -24,6 +24,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -32,6 +33,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -100,12 +102,16 @@ func init() {
 		PersistOnFailure: false,
 	}
 
+	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
+
+	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()
+
 	// Call to refresh will refresh names in cache. If you pass true, it will also
 	// remove cached names not looked up since the last call to Refresh. It is a good idea
 	// to call this method on a regular interval.
 	go func() {
 		var t *time.Ticker
-		if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
+		if containers {
 			t = time.NewTicker(1 * time.Minute)
 		} else {
 			t = time.NewTicker(10 * time.Minute)
@@ -172,6 +178,13 @@ func minioConfigToConsoleFeatures() {
 			os.Setenv("CONSOLE_LOG_QUERY_AUTH_TOKEN", value)
 		}
 	}
+	// pass the console subpath configuration
+	if value := env.Get(config.EnvMinIOBrowserRedirectURL, ""); value != "" {
+		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
+		if subPath != SlashSeparator {
+			os.Setenv("CONSOLE_SUBPATH", subPath)
+		}
+	}
 	// Enable if prometheus URL is set.
 	if value := env.Get("MINIO_PROMETHEUS_URL", ""); value != "" {
 		os.Setenv("CONSOLE_PROMETHEUS_URL", value)
@@ -184,23 +197,23 @@ func minioConfigToConsoleFeatures() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
 	// if IDP is enabled, set IDP environment variables
-	if globalOpenIDConfig.URL != nil {
-		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.URL.String())
-		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ClientID)
-		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ClientSecret)
+	if globalOpenIDConfig.ProviderCfgs[config.Default] != nil {
+		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.ProviderCfgs[config.Default].URL.String())
+		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ProviderCfgs[config.Default].ClientID)
+		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ProviderCfgs[config.Default].ClientSecret)
 		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
-		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ClientID)
-		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.DiscoveryDoc.ScopesSupported, ","))
-		if globalOpenIDConfig.ClaimUserinfo {
+		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ProviderCfgs[config.Default].ClientID)
+		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.ProviderCfgs[config.Default].DiscoveryDoc.ScopesSupported, ","))
+		if globalOpenIDConfig.ProviderCfgs[config.Default].ClaimUserinfo {
 			os.Setenv("CONSOLE_IDP_USERINFO", config.EnableOn)
 		}
-		if globalOpenIDConfig.RedirectURIDynamic {
+		if globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURIDynamic {
 			// Enable dynamic redirect-uri's based on incoming 'host' header,
 			// Overrides any other callback URL.
 			os.Setenv("CONSOLE_IDP_CALLBACK_DYNAMIC", config.EnableOn)
 		}
-		if globalOpenIDConfig.RedirectURI != "" {
-			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.RedirectURI)
+		if globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURI != "" {
+			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURI)
 		} else {
 			os.Setenv("CONSOLE_IDP_CALLBACK", getConsoleEndpoints()[0]+"/oauth_callback")
 		}
@@ -482,6 +495,13 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 	key := envTokens[0]
 	val := envTokens[1]
 
+	if strings.HasPrefix(key, "#") {
+		// Skip commented lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+
 	// Remove quotes from the value if found
 	if len(val) >= 2 {
 		quote := val[0]
@@ -625,12 +645,11 @@ func handleCommonEnvVars() {
 			}
 			// Look for if URL has invalid values and return error.
 			if !((u.Scheme == "http" || u.Scheme == "https") &&
-				(u.Path == "/" || u.Path == "") && u.Opaque == "" &&
+				u.Opaque == "" &&
 				!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
 				err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable")
 			}
-			u.Path = "" // remove any path component such as `/`
 			globalBrowserRedirectURL = u
 		}
 	}
@@ -664,8 +683,6 @@ func handleCommonEnvVars() {
 		globalRootDiskThreshold = size
 	}
 
-	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
-
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
@@ -796,7 +813,29 @@ func handleCommonEnvVars() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		certificate, err := tls.LoadX509KeyPair(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""))
+		// Manually load the certificate and private key into memory.
+		// We need to check whether the private key is encrypted, and
+		// if so, decrypt it using the user-provided password.
+		certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
+		}
+		keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
+		}
+		privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+		if len(rest) != 0 {
+			logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
+		}
+		if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+			keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+			if err != nil {
+				logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
+			}
+			keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+		}
+		certificate, err := tls.X509KeyPair(certBytes, keyBytes)
 		if err != nil {
 			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
 		}

cmd/common-main_test.go

@@ -134,6 +134,24 @@ export MINIO_ROOT_PASSWORD=minio123`,
 			true,
 			nil,
 		},
+		{
+			`
+# MINIO_ROOT_USER=minioadmin
+# MINIO_ROOT_PASSWORD=minioadmin
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
 	}
 	for _, testCase := range testCases {
 		testCase := testCase
@@ -153,6 +171,11 @@ export MINIO_ROOT_PASSWORD=minio123`,
 			if err == nil && testCase.expectedErr {
 				t.Error(errors.New("expected error, found success"))
 			}
+
+			if len(ekvs) != len(testCase.expectedEkvs) {
+				t.Errorf("expected %v keys, got %v keys", len(testCase.expectedEkvs), len(ekvs))
+			}
+
 			if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) {
 				t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs)
 			}

cmd/config-current.go

@@ -94,8 +94,9 @@ func initHelp() {
 			Description: "federate multiple clusters for IAM and Bucket DNS",
 		},
 		config.HelpKV{
-			Key:         config.IdentityOpenIDSubSys,
-			Description: "enable OpenID SSO support",
+			Key:             config.IdentityOpenIDSubSys,
+			Description:     "enable OpenID SSO support",
+			MultipleTargets: true,
 		},
 		config.HelpKV{
 			Key:         config.IdentityLDAPSubSys,
@@ -314,7 +315,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			etcdClnt.Close()
 		}
 	case config.IdentityOpenIDSubSys:
-		if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
+		if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys],
 			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 			return err
 		}
@@ -484,22 +485,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
 	})
 
-	if globalIsErasure && objAPI != nil {
-		setDriveCounts := objAPI.SetDriveCounts()
-		for i, setDriveCount := range setDriveCounts {
-			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
-			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
-				break
-			}
-			// if we validated all setDriveCounts and it was successful
-			// proceed to store the correct storage class globally.
-			if i == len(setDriveCounts)-1 {
-				globalStorageClass.Update(sc)
-			}
-		}
-	}
-
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
 		if globalIsGateway {
@@ -532,7 +517,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
-	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
+	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys],
 		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
@@ -543,8 +528,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OPA: %w", err))
 	}
-
-	globalOpenIDValidators = getOpenIDValidators(globalOpenIDConfig)
 	globalPolicyOPA = opa.New(opaCfg)
 
 	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
@@ -670,6 +653,22 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
 		}
+	case config.StorageClassSubSys:
+		if globalIsErasure && objAPI != nil {
+			setDriveCounts := objAPI.SetDriveCounts()
+			for i, setDriveCount := range setDriveCounts {
+				sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
+				if err != nil {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+					break
+				}
+				// if we validated all setDriveCounts and it was successful
+				// proceed to store the correct storage class globally.
+				if i == len(setDriveCounts)-1 {
+					globalStorageClass.Update(sc)
+				}
+			}
+		}
 	}
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
@@ -807,17 +806,3 @@ func loadConfig(objAPI ObjectLayer) error {
 
 	return nil
 }
-
-// getOpenIDValidators - returns ValidatorList which contains
-// enabled providers in server config.
-// A new authentication provider is added like below
-// * Add a new provider in pkg/iam/openid package.
-func getOpenIDValidators(cfg openid.Config) *openid.Validators {
-	validators := openid.NewValidators()
-
-	if cfg.Enabled {
-		validators.Add(&cfg)
-	}
-
-	return validators
-}

cmd/config-migrate.go

@@ -2649,7 +2649,6 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 
 	cfg.Version = "31"
 	cfg.OpenID = openid.Config{}
-	cfg.OpenID.JWKS.URL = &xnet.URL{}
 
 	cfg.Policy.OPA = opa.Args{
 		URL:       &xnet.URL{},

cmd/data-scanner.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -37,6 +38,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/heal"
@@ -103,6 +105,63 @@ func (s *safeDuration) Get() time.Duration {
 	return s.t
 }
 
+func getCycleScanMode(currentCycle, bitrotStartCycle uint64, bitrotStartTime time.Time) madmin.HealScanMode {
+	bitrotCycle := globalHealConfig.BitrotScanCycle()
+	switch bitrotCycle {
+	case -1:
+		return madmin.HealNormalScan
+	case 0:
+		return madmin.HealDeepScan
+	}
+
+	if currentCycle-bitrotStartCycle < healObjectSelectProb {
+		return madmin.HealDeepScan
+	}
+
+	if time.Since(bitrotStartTime) > bitrotCycle {
+		return madmin.HealDeepScan
+	}
+
+	return madmin.HealNormalScan
+}
+
+type backgroundHealInfo struct {
+	BitrotStartTime  time.Time           `json:"bitrotStartTime"`
+	BitrotStartCycle uint64              `json:"bitrotStartCycle"`
+	CurrentScanMode  madmin.HealScanMode `json:"currentScanMode"`
+}
+
+func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo {
+	// Get last healing information
+	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
+	if err != nil {
+		if !errors.Is(err, errConfigNotFound) {
+			logger.LogIf(ctx, err)
+		}
+		return backgroundHealInfo{}
+	}
+	var info backgroundHealInfo
+	err = json.Unmarshal(buf, &info)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return backgroundHealInfo{}
+	}
+	return info
+}
+
+func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) {
+	b, err := json.Marshal(info)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+	// Get last healing information
+	err = saveConfig(ctx, objAPI, backgroundHealInfoPath, b)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+}
+
 // runDataScanner will start a data scanner.
 // The function will block until the context is canceled.
 // There should only ever be one scanner running per cluster.
@@ -145,12 +204,24 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 				console.Debugln("starting scanner cycle")
 			}
 
+			bgHealInfo := readBackgroundHealInfo(ctx, objAPI)
+			scanMode := getCycleScanMode(nextBloomCycle, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
+			if bgHealInfo.CurrentScanMode != scanMode {
+				newHealInfo := bgHealInfo
+				newHealInfo.CurrentScanMode = scanMode
+				if scanMode == madmin.HealDeepScan {
+					newHealInfo.BitrotStartTime = time.Now().UTC()
+					newHealInfo.BitrotStartCycle = nextBloomCycle
+				}
+				saveBackgroundHealInfo(ctx, objAPI, newHealInfo)
+			}
+
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle))
+			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle), scanMode)
 			logger.LogIf(ctx, err)
 			if err == nil {
 				// Store new cycle...
@@ -182,6 +253,7 @@ type folderScanner struct {
 	dataUsageScannerDebug bool
 	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
+	scanMode              madmin.HealScanMode
 
 	disks       []StorageAPI
 	disksQuorum int
@@ -250,7 +322,7 @@ var globalScannerStats scannerStats
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
@@ -279,6 +351,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		dataUsageScannerDebug: intDataUpdateTracker.debug,
 		healFolderInclude:     0,
 		healObjectSelect:      0,
+		scanMode:              scanMode,
 		updates:               cache.Info.updates,
 	}
 
@@ -482,12 +555,15 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				debug:       f.dataUsageScannerDebug,
 				lifeCycle:   activeLifeCycle,
 				replication: replicationCfg,
-				heal:        thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
+
+			item.heal.enabled = thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure
+			item.heal.bitrot = f.scanMode == madmin.HealDeepScan
+
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
 			// healing attempt on this drive.
-			item.heal = item.heal && f.healObjectSelect > 0
+			item.heal.enabled = item.heal.enabled && f.healObjectSelect > 0
 
 			sz, err := f.getSize(item)
 			if err != nil {
@@ -821,8 +897,11 @@ type scannerItem struct {
 	replication replicationConfig
 	lifeCycle   *lifecycle.Lifecycle
 	Typ         fs.FileMode
-	heal        bool // Has the object been selected for heal check?
-	debug       bool
+	heal        struct {
+		enabled bool
+		bitrot  bool
+	} // Has the object been selected for heal check?
+	debug bool
 }
 
 type sizeSummary struct {
@@ -874,9 +953,13 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
 			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
 		}
 	}
+	scanMode := madmin.HealNormalScan
+	if i.heal.bitrot {
+		scanMode = madmin.HealDeepScan
+	}
 	healOpts := madmin.HealOpts{
 		Remove:   healDeleteDangling,
-		ScanMode: globalHealConfig.ScanMode(),
+		ScanMode: scanMode,
 	}
 	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
 	if err != nil && !errors.Is(err, NotImplemented{}) {
@@ -901,20 +984,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
 	versionID := oi.VersionID
-	action := i.lifeCycle.ComputeAction(
-		lifecycle.ObjectOpts{
-			Name:             i.objectPath(),
-			UserTags:         oi.UserTags,
-			ModTime:          oi.ModTime,
-			VersionID:        oi.VersionID,
-			DeleteMarker:     oi.DeleteMarker,
-			IsLatest:         oi.IsLatest,
-			NumVersions:      oi.NumVersions,
-			SuccessorModTime: oi.SuccessorModTime,
-			RestoreOngoing:   oi.RestoreOngoing,
-			RestoreExpires:   oi.RestoreExpires,
-			TransitionStatus: oi.TransitionedObject.Status,
-		})
+	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
+	action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
 	if i.debug {
 		if versionID != "" {
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
@@ -1040,7 +1111,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
-		if i.heal {
+		if i.heal.enabled {
 			size = i.applyHealing(ctx, o, oi)
 		}
 		// replicate only if lifecycle rules are not applied.
@@ -1049,7 +1120,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }
 
-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj ObjectInfo, debug bool) (action lifecycle.Action) {
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo, debug bool) (action lifecycle.Action) {
 	action = lc.ComputeAction(obj.ToLifecycleOpts())
 	if debug {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
@@ -1065,18 +1136,15 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj Ob
 		if obj.VersionID == "" {
 			return lifecycle.NoneAction
 		}
-		if rcfg, _ := globalBucketObjectLockSys.Get(obj.Bucket); rcfg.LockEnabled {
-			locked := enforceRetentionForDeletion(ctx, obj)
-			if locked {
-				if debug {
-					if obj.VersionID != "" {
-						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
-					} else {
-						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
-					}
+		if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
+			if debug {
+				if obj.VersionID != "" {
+					console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
+				} else {
+					console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 				}
-				return lifecycle.NoneAction
 			}
+			return lifecycle.NoneAction
 		}
 	}
 
@@ -1195,32 +1263,31 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 		roi.OpType = replication.ExistingObjectReplicationType
 	}
 
-	if roi.TargetStatuses != nil {
-		if sizeS.replTargetStats == nil {
-			sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
+	if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 {
+		sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
+	}
+
+	for arn, tgtStatus := range roi.TargetStatuses {
+		tgtSizeS, ok := sizeS.replTargetStats[arn]
+		if !ok {
+			tgtSizeS = replTargetSizeSummary{}
 		}
-		for arn, tgtStatus := range roi.TargetStatuses {
-			tgtSizeS, ok := sizeS.replTargetStats[arn]
-			if !ok {
-				tgtSizeS = replTargetSizeSummary{}
-			}
-			switch tgtStatus {
-			case replication.Pending:
-				tgtSizeS.pendingCount++
-				tgtSizeS.pendingSize += oi.Size
-				sizeS.pendingCount++
-				sizeS.pendingSize += oi.Size
-			case replication.Failed:
-				tgtSizeS.failedSize += oi.Size
-				tgtSizeS.failedCount++
-				sizeS.failedSize += oi.Size
-				sizeS.failedCount++
-			case replication.Completed, "COMPLETE":
-				tgtSizeS.replicatedSize += oi.Size
-				sizeS.replicatedSize += oi.Size
-			}
-			sizeS.replTargetStats[arn] = tgtSizeS
+		switch tgtStatus {
+		case replication.Pending:
+			tgtSizeS.pendingCount++
+			tgtSizeS.pendingSize += oi.Size
+			sizeS.pendingCount++
+			sizeS.pendingSize += oi.Size
+		case replication.Failed:
+			tgtSizeS.failedSize += oi.Size
+			tgtSizeS.failedCount++
+			sizeS.failedSize += oi.Size
+			sizeS.failedCount++
+		case replication.Completed, replication.CompletedLegacy:
+			tgtSizeS.replicatedSize += oi.Size
+			sizeS.replicatedSize += oi.Size
 		}
+		sizeS.replTargetStats[arn] = tgtSizeS
 	}
 
 	switch oi.ReplicationStatus {

cmd/data-usage-cache.go

@@ -535,12 +535,13 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	}
 	flat := d.flatten(*e)
 	dui := DataUsageInfo{
-		LastUpdate:        d.Info.LastUpdate,
-		ObjectsTotalCount: flat.Objects,
-		ObjectsTotalSize:  uint64(flat.Size),
-		BucketsCount:      uint64(len(e.Children)),
-		BucketsUsage:      d.bucketsUsageInfo(buckets),
-		TierStats:         d.tiersUsageInfo(buckets),
+		LastUpdate:         d.Info.LastUpdate,
+		ObjectsTotalCount:  flat.Objects,
+		VersionsTotalCount: flat.Versions,
+		ObjectsTotalSize:   uint64(flat.Size),
+		BucketsCount:       uint64(len(e.Children)),
+		BucketsUsage:       d.bucketsUsageInfo(buckets),
+		TierStats:          d.tiersUsageInfo(buckets),
 	}
 	return dui
 }
@@ -788,6 +789,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		flat := d.flatten(*e)
 		bui := BucketUsageInfo{
 			Size:                 uint64(flat.Size),
+			VersionsCount:        flat.Versions,
 			ObjectsCount:         flat.Objects,
 			ObjectSizesHistogram: flat.ObjSizes.toMap(),
 		}

cmd/data-usage-utils.go

@@ -60,6 +60,7 @@ type BucketUsageInfo struct {
 
 	ObjectsCount         uint64                           `json:"objectsCount"`
 	ObjectSizesHistogram map[string]uint64                `json:"objectsSizesHistogram"`
+	VersionsCount        uint64                           `json:"versionsCount"`
 	ReplicaSize          uint64                           `json:"objectReplicaTotalSize"`
 	ReplicationInfo      map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
 }
@@ -73,6 +74,9 @@ type DataUsageInfo struct {
 	// Objects total count across all buckets
 	ObjectsTotalCount uint64 `json:"objectsCount"`
 
+	// Objects total count across all buckets
+	VersionsTotalCount uint64 `json:"versionsCount"`
+
 	// Objects total size across all buckets
 	ObjectsTotalSize uint64                           `json:"objectsTotalSize"`
 	ReplicationInfo  map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`

cmd/data-usage.go

@@ -35,6 +35,8 @@ const (
 	dataUsageBloomName     = ".bloomcycle.bin"
 	dataUsageBloomNamePath = bucketMetaPrefix + SlashSeparator + dataUsageBloomName
 
+	backgroundHealInfoPath = bucketMetaPrefix + SlashSeparator + ".background-heal.json"
+
 	dataUsageCacheName = ".usage-cache.bin"
 )
 

cmd/data-usage_test.go

@@ -67,7 +67,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return
 	}
 
-	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
+	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -178,7 +178,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize)
+		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize, 0)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -289,7 +289,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return
 	}
-	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
+	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -423,7 +423,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize)
+		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize, 0)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -575,7 +575,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return
 	}
-	want, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
+	want, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/disk-cache-utils.go

@@ -19,19 +19,17 @@ package cmd
 
 import (
 	"container/list"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"math"
 	"os"
-	"path"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/etag"
 )
 
 // CacheStatusType - whether the request was served from cache.
@@ -234,77 +232,85 @@ func isCacheEncrypted(meta map[string]string) bool {
 
 // decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
 func decryptCacheObjectETag(info *ObjectInfo) error {
-	// Directories are never encrypted.
 	if info.IsDir {
-		return nil
+		return nil // Directories are never encrypted.
 	}
-	encrypted := crypto.S3.IsEncrypted(info.UserDefined) && isCacheEncrypted(info.UserDefined)
 
-	switch {
-	case encrypted:
-		if globalCacheKMS == nil {
-			return errKMSNotConfigured
+	// Depending on the SSE type we handle ETags slightly
+	// differently. ETags encrypted with SSE-S3 must be
+	// decrypted first, since the client expects that
+	// a single-part SSE-S3 ETag is equal to the content MD5.
+	//
+	// For all other SSE types, the ETag is not the content MD5.
+	// Therefore, we don't decrypt but only format it.
+	switch kind, ok := crypto.IsEncrypted(info.UserDefined); {
+	case ok && kind == crypto.S3 && isCacheEncrypted(info.UserDefined):
+		ETag, err := etag.Parse(info.ETag)
+		if err != nil {
+			return err
 		}
-		if len(info.Parts) > 0 { // multipart ETag is not encrypted since it is not md5sum
+		if !ETag.IsEncrypted() {
+			info.ETag = ETag.Format().String()
 			return nil
 		}
-		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(info.UserDefined)
-		if err != nil {
-			return err
-		}
-		extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{info.Bucket: path.Join(info.Bucket, info.Name)})
-		if err != nil {
-			return err
-		}
-		var objectKey crypto.ObjectKey
-		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), info.Bucket, info.Name); err != nil {
-			return err
-		}
-		etagStr := tryDecryptETag(objectKey[:], info.ETag, false)
-		// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
-		etag, err := hex.DecodeString(etagStr)
-		if err != nil {
-			return err
-		}
-		info.ETag = string(etag)
-		return nil
-	}
 
+		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, info.UserDefined, info.Bucket, info.Name)
+		if err != nil {
+			return err
+		}
+		ETag, err = etag.Decrypt(key[:], ETag)
+		if err != nil {
+			return err
+		}
+		info.ETag = ETag.Format().String()
+	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(info.UserDefined):
+		ETag, err := etag.Parse(info.ETag)
+		if err != nil {
+			return err
+		}
+		info.ETag = ETag.Format().String()
+	}
 	return nil
 }
 
 // decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
 func decryptCachePartETags(c *cacheMeta) ([]string, error) {
-	var partETags []string
-	encrypted := crypto.S3.IsEncrypted(c.Meta) && isCacheEncrypted(c.Meta)
-
-	switch {
-	case encrypted:
-		if globalCacheKMS == nil {
-			return partETags, errKMSNotConfigured
-		}
-		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(c.Meta)
+	// Depending on the SSE type we handle ETags slightly
+	// differently. ETags encrypted with SSE-S3 must be
+	// decrypted first, since the client expects that
+	// a single-part SSE-S3 ETag is equal to the content MD5.
+	//
+	// For all other SSE types, the ETag is not the content MD5.
+	// Therefore, we don't decrypt but only format it.
+	switch kind, ok := crypto.IsEncrypted(c.Meta); {
+	case ok && kind == crypto.S3 && isCacheEncrypted(c.Meta):
+		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, c.Meta, c.Bucket, c.Object)
 		if err != nil {
-			return partETags, err
-		}
-		extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{c.Bucket: path.Join(c.Bucket, c.Object)})
-		if err != nil {
-			return partETags, err
-		}
-		var objectKey crypto.ObjectKey
-		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), c.Bucket, c.Object); err != nil {
-			return partETags, err
+			return nil, err
 		}
+		etags := make([]string, 0, len(c.PartETags))
 		for i := range c.PartETags {
-			etagStr := tryDecryptETag(objectKey[:], c.PartETags[i], false)
-			// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
-			etag, err := hex.DecodeString(etagStr)
+			ETag, err := etag.Parse(c.PartETags[i])
 			if err != nil {
-				return []string{}, err
+				return nil, err
 			}
-			partETags = append(partETags, string(etag))
+			ETag, err = etag.Decrypt(key[:], ETag)
+			if err != nil {
+				return nil, err
+			}
+			etags = append(etags, ETag.Format().String())
 		}
-		return partETags, nil
+		return etags, nil
+	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(c.Meta):
+		etags := make([]string, 0, len(c.PartETags))
+		for i := range c.PartETags {
+			ETag, err := etag.Parse(c.PartETags[i])
+			if err != nil {
+				return nil, err
+			}
+			etags = append(etags, ETag.Format().String())
+		}
+		return etags, nil
 	default:
 		return c.PartETags, nil
 	}

cmd/encryption-v1.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bufio"
+	"context"
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
@@ -35,6 +36,7 @@ import (
 
 	"github.com/minio/kes"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
@@ -71,18 +73,132 @@ const (
 )
 
 // KMSKeyID returns in AWS compatible KMS KeyID() format.
-func (o ObjectInfo) KMSKeyID() string {
-	if len(o.UserDefined) == 0 {
+func (o *ObjectInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) }
+
+// KMSKeyID returns in AWS compatible KMS KeyID() format.
+func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) }
+
+// kmsKeyIDFromMetadata returns any AWS S3 KMS key ID in the
+// metadata, if any. It returns an empty ID if no key ID is
+// present.
+func kmsKeyIDFromMetadata(metadata map[string]string) string {
+	const ARNPrefix = "arn:aws:kms:"
+	if len(metadata) == 0 {
 		return ""
 	}
-	kmsID, ok := o.UserDefined[crypto.MetaKeyID]
+	kmsID, ok := metadata[crypto.MetaKeyID]
 	if !ok {
 		return ""
 	}
-	if strings.HasPrefix(kmsID, "arn:aws:kms:") {
+	if strings.HasPrefix(kmsID, ARNPrefix) {
 		return kmsID
 	}
-	return "arn:aws:kms:" + kmsID
+	return ARNPrefix + kmsID
+}
+
+// DecryptETags decryptes the ETag of all ObjectInfos using the KMS.
+//
+// It adjusts the size of all encrypted objects since encrypted
+// objects are slightly larger due to encryption overhead.
+// Further, it decrypts all single-part SSE-S3 encrypted objects
+// and formats ETags of SSE-C / SSE-KMS encrypted objects to
+// be AWS S3 compliant.
+//
+// DecryptETags uses a KMS bulk decryption API, if available, which
+// is more efficient than decrypting ETags sequentually.
+func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {
+	const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
+	var (
+		metadata = make([]map[string]string, 0, BatchSize)
+		buckets  = make([]string, 0, BatchSize)
+		names    = make([]string, 0, BatchSize)
+	)
+	for len(objects) > 0 {
+		N := BatchSize
+		if len(objects) < BatchSize {
+			N = len(objects)
+		}
+		batch := objects[:N]
+
+		// We have to decrypt only ETags of SSE-S3 single-part
+		// objects.
+		// Therefore, we remember which objects (there index)
+		// in the current batch are single-part SSE-S3 objects.
+		metadata = metadata[:0:N]
+		buckets = buckets[:0:N]
+		names = names[:0:N]
+		SSES3SinglePartObjects := make(map[int]bool)
+		for i, object := range batch {
+			if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
+				SSES3SinglePartObjects[i] = true
+
+				metadata = append(metadata, object.UserDefined)
+				buckets = append(buckets, object.Bucket)
+				names = append(names, object.Name)
+			}
+		}
+
+		// If there are no SSE-S3 single-part objects
+		// we can skip the decryption process. However,
+		// we still have to adjust the size and ETag
+		// of SSE-C and SSE-KMS objects.
+		if len(SSES3SinglePartObjects) == 0 {
+			for i := range batch {
+				size, err := batch[i].GetActualSize()
+				if err != nil {
+					return err
+				}
+				batch[i].Size = size
+
+				if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok {
+					ETag, err := etag.Parse(batch[i].ETag)
+					if err != nil {
+						return err
+					}
+					batch[i].ETag = ETag.Format().String()
+				}
+			}
+			objects = objects[N:]
+			continue
+		}
+
+		// There is at least one SSE-S3 single-part object.
+		// For all SSE-S3 single-part objects we have to
+		// fetch their decryption keys. We do this using
+		// a Bulk-Decryption API call, if available.
+		keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)
+		if err != nil {
+			return err
+		}
+
+		// Now, we have to decrypt the ETags of SSE-S3 single-part
+		// objects and adjust the size and ETags of all encrypted
+		// objects.
+		for i := range batch {
+			size, err := batch[i].GetActualSize()
+			if err != nil {
+				return err
+			}
+			batch[i].Size = size
+
+			if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok {
+				ETag, err := etag.Parse(batch[i].ETag)
+				if err != nil {
+					return err
+				}
+				if SSES3SinglePartObjects[i] && ETag.IsEncrypted() {
+					ETag, err = etag.Decrypt(keys[0][:], ETag)
+					if err != nil {
+						return err
+					}
+					keys = keys[1:]
+				}
+				batch[i].ETag = ETag.Format().String()
+			}
+		}
+		objects = objects[N:]
+	}
+	return nil
 }
 
 // isMultipart returns true if the current object is

cmd/erasure-bucket.go

@@ -39,8 +39,10 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Verify if bucket is valid.
-	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
-		return BucketNameInvalid{Bucket: bucket}
+	if !isMinioMetaBucketName(bucket) {
+		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
+			return BucketNameInvalid{Bucket: bucket}
+		}
 	}
 
 	storageDisks := er.getDisks()
@@ -53,6 +55,11 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		g.Go(func() error {
 			if storageDisks[index] != nil {
 				if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
+					if opts.ForceCreate && errors.Is(err, errVolumeExists) {
+						// No need to return error when force create was
+						// requested.
+						return nil
+					}
 					if !errors.Is(err, errVolumeExists) {
 						logger.LogIf(ctx, err)
 					}

cmd/erasure-healing-common.go

@@ -31,7 +31,7 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 	groupNano := group.Nanoseconds()
 	// Ignore the uuid sentinel and count the rest.
 	for _, t := range times {
-		if t.Equal(timeSentinel) {
+		if t.Equal(timeSentinel) || t.IsZero() {
 			continue
 		}
 		nano := t.UnixNano()

cmd/erasure-healing.go

@@ -60,7 +60,8 @@ func (fi FileInfo) DataShardFixed() bool {
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
 func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
-	result madmin.HealResultItem, err error) {
+	result madmin.HealResultItem, err error,
+) {
 	if !opts.DryRun {
 		defer NSUpdated(bucket, slashSeparator)
 	}

cmd/erasure-healing_test.go

@@ -758,8 +758,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 			t.Fatalf("Failed to getLatestFileInfo - %v", err)
 		}
 
+		fi.DiskMTime = time.Time{}
+		nfi.DiskMTime = time.Time{}
 		if !reflect.DeepEqual(fi, nfi) {
-			t.Fatalf("FileInfo not equal after healing")
+			t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 		}
 
 		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
@@ -784,8 +786,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 			t.Fatalf("Failed to getLatestFileInfo - %v", err)
 		}
 
+		fi.DiskMTime = time.Time{}
+		nfi.DiskMTime = time.Time{}
 		if !reflect.DeepEqual(fi, nfi) {
-			t.Fatalf("FileInfo not equal after healing")
+			t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 		}
 
 		// Test 4: checks if HealObject returns an error when xl.meta is not found
@@ -904,6 +908,8 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
+	fi.DiskMTime = time.Time{}
+	nfi1.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi1) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
@@ -925,6 +931,8 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
+	fi.DiskMTime = time.Time{}
+	nfi2.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi2) {
 		t.Fatalf("FileInfo not equal after healing")
 	}

cmd/erasure-object.go

@@ -33,6 +33,7 @@ import (
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
@@ -80,12 +81,26 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	}
 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
-	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, true)
 
-	// get Quorum for this object
+	var metaArr []FileInfo
+	var errs []error
+
+	// Read metadata associated with the object from all disks.
+	if srcOpts.VersionID != "" {
+		metaArr, errs = readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, true)
+	} else {
+		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true)
+	}
+
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		return oi, toObjectErr(err, srcBucket, srcObject)
+		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(srcBucket, minioMetaBucket) {
+			_, derr := er.deleteIfDangling(ctx, srcBucket, srcObject, metaArr, errs, nil, srcOpts)
+			if derr != nil {
+				err = derr
+			}
+		}
+		return ObjectInfo{}, toObjectErr(err, srcBucket, srcObject)
 	}
 
 	// List all online disks.
@@ -111,9 +126,20 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		// preserve destination versionId if specified.
 		if versionID == "" {
 			versionID = mustGetUUID()
+			fi.IsLatest = true // we are creating a new version so this is latest.
 		}
 		modTime = UTCNow()
 	}
+
+	// If the data is not inlined, we may end up incorrectly
+	// inlining the data here, that leads to an inconsistent
+	// situation where some objects are were not inlined
+	// were now inlined, make sure to `nil` the Data such
+	// that xl.meta is written as expected.
+	if !fi.InlineData() {
+		fi.Data = nil
+	}
+
 	fi.VersionID = versionID // set any new versionID we might have created
 	fi.ModTime = modTime     // set modTime for the new versionID
 	if !dstOpts.MTime.IsZero() {
@@ -129,6 +155,14 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 			metaArr[index].ModTime = modTime
 			metaArr[index].VersionID = versionID
 			metaArr[index].Metadata = srcInfo.UserDefined
+			if !metaArr[index].InlineData() {
+				// If the data is not inlined, we may end up incorrectly
+				// inlining the data here, that leads to an inconsistent
+				// situation where some objects are were not inlined
+				// were now inlined, make sure to `nil` the Data such
+				// that xl.meta is written as expected.
+				metaArr[index].Data = nil
+			}
 		}
 	}
 
@@ -137,8 +171,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
-	// we are adding a new version to this object under the namespace lock, so this is the latest version.
-	fi.IsLatest = true
 	return fi.ToObjectInfo(srcBucket, srcObject), nil
 }
 
@@ -418,11 +450,90 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 	return m, err
 }
 
+func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData bool) ([]FileInfo, []error) {
+	metadataArray := make([]*xlMetaV2, len(disks))
+	metaFileInfos := make([]FileInfo, len(metadataArray))
+	metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(disks))
+
+	g := errgroup.WithNErrs(len(disks))
+	// Read `xl.meta` in parallel across disks.
+	for index := range disks {
+		index := index
+		g.Go(func() (err error) {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			rf, err := disks[index].ReadXL(ctx, bucket, object, readData)
+			if err != nil {
+				if !IsErr(err, []error{
+					errFileNotFound,
+					errVolumeNotFound,
+					errFileVersionNotFound,
+					errDiskNotFound,
+				}...) {
+					logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
+						disks[index], bucket, object, err),
+						disks[index].String())
+				}
+				return err
+			}
+
+			var xl xlMetaV2
+			if err = xl.LoadOrConvert(rf.Buf); err != nil {
+				return err
+			}
+			metadataArray[index] = &xl
+			metaFileInfos[index] = FileInfo{
+				DiskMTime: rf.DiskMTime,
+			}
+			return nil
+		}, index)
+	}
+
+	errs := g.Wait()
+	for index := range metadataArray {
+		if metadataArray[index] != nil {
+			metadataShallowVersions[index] = metadataArray[index].versions
+		}
+	}
+
+	readQuorum := (len(disks) + 1) / 2
+	merged := mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)
+	for index := range metadataArray {
+		if metadataArray[index] == nil {
+			continue
+		}
+
+		metadataArray[index].versions = merged
+
+		// make sure to preserve this for diskmtime based healing bugfix.
+		diskMTime := metaFileInfos[index].DiskMTime
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, "")
+		if errs[index] == nil {
+			versionID := metaFileInfos[index].VersionID
+			if versionID == "" {
+				versionID = nullVersionID
+			}
+			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
+			metaFileInfos[index].DiskMTime = diskMTime
+		}
+	}
+
+	// Return all the metadata.
+	return metaFileInfos, errs
+}
+
 func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
 	disks := er.getDisks()
 
+	var errs []error
+
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
+	if opts.VersionID != "" {
+		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
+	} else {
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, readData)
+	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
@@ -493,11 +604,6 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
-	if opts.VersionID != "" && !fi.VersionPurgeStatus().Empty() {
-		// Make sure to return object info to provide extra information.
-		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
-	}
-
 	if fi.Deleted {
 		if opts.VersionID == "" || opts.DeleteMarker {
 			return objInfo, toObjectErr(errFileNotFound, bucket, object)
@@ -1227,9 +1333,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	var lc *lifecycle.Lifecycle
+	var rcfg lock.Retention
 	if opts.Expiration.Expire {
 		// Check if the current bucket has a configured lifecycle policy
 		lc, _ = globalLifecycleSys.Get(bucket)
+		rcfg, _ = globalBucketObjectLockSys.Get(bucket)
 	}
 
 	// expiration attempted on a bucket with no lifecycle
@@ -1274,7 +1382,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	if opts.Expiration.Expire {
-		action := evalActionFromLifecycle(ctx, *lc, goi, false)
+		action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
 		var isErr bool
 		switch action {
 		case lifecycle.NoneAction:
@@ -1321,10 +1429,21 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if opts.VersionPurgeStatus() == Complete {
 			markDelete = false
 		}
-		// determine if the version represents an object delete
-		// deleteMarker = true
-		if versionFound && !goi.DeleteMarker { // implies a versioned delete of object
-			deleteMarker = false
+
+		// Version is found but we do not wish to create more delete markers
+		// now, since VersionPurgeStatus() is already set, we can let the
+		// lower layers decide this. This fixes a regression that was introduced
+		// in PR #14555 where !VersionPurgeStatus.Empty() is automatically
+		// considered as Delete marker true to avoid listing such objects by
+		// regular ListObjects() calls. However for delete replication this
+		// ends up being a problem because "upon" a successful delete this
+		// ends up creating a new delete marker that is spurious and unnecessary.
+		if versionFound {
+			if !goi.VersionPurgeStatus.Empty() {
+				deleteMarker = false
+			} else if !goi.DeleteMarker { // implies a versioned delete of object
+				deleteMarker = false
+			}
 		}
 	}
 
@@ -1427,11 +1546,24 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 	disks := er.getDisks()
 
+	var metaArr []FileInfo
+	var errs []error
+
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
+	if opts.VersionID != "" {
+		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
+	} else {
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false)
+	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
+		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			if derr != nil {
+				err = derr
+			}
+		}
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -1487,11 +1619,24 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	disks := er.getDisks()
 
+	var metaArr []FileInfo
+	var errs []error
+
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
+	if opts.VersionID != "" {
+		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
+	} else {
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false)
+	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
+		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			if derr != nil {
+				err = derr
+			}
+		}
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 

cmd/erasure-server-pool-decom.go

@@ -24,12 +24,16 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/env"
 )
 
 // PoolDecommissionInfo currently decommissioning information
@@ -141,7 +145,7 @@ func (p *poolMeta) returnResumablePools(n int) []PoolStatus {
 }
 
 func (p *poolMeta) DecommissionComplete(idx int) bool {
-	if p.Pools[idx].Decommission != nil {
+	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Complete {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.Complete = true
 		p.Pools[idx].Decommission.Failed = false
@@ -152,7 +156,7 @@ func (p *poolMeta) DecommissionComplete(idx int) bool {
 }
 
 func (p *poolMeta) DecommissionFailed(idx int) bool {
-	if p.Pools[idx].Decommission != nil {
+	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Failed {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.StartTime = time.Time{}
 		p.Pools[idx].Decommission.Complete = false
@@ -164,7 +168,7 @@ func (p *poolMeta) DecommissionFailed(idx int) bool {
 }
 
 func (p *poolMeta) DecommissionCancel(idx int) bool {
-	if p.Pools[idx].Decommission != nil {
+	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Canceled {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.StartTime = time.Time{}
 		p.Pools[idx].Decommission.Complete = false
@@ -306,9 +310,31 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 		specifiedPools[pool.endpoints.CmdLine] = idx
 	}
 
+	replaceScheme := func(k string) string {
+		// This is needed as fallback when users are changeing
+		// from http->https or https->http, we need to verify
+		// both because MinIO remembers the command-line in
+		// "exact" order - as long as this order is not disturbed
+		// we allow changing the "scheme" i.e internode communication
+		// from plain-text to TLS or from TLS to plain-text.
+		if strings.HasPrefix(k, "http://") {
+			k = strings.ReplaceAll(k, "http://", "https://")
+		} else if strings.HasPrefix(k, "https://") {
+			k = strings.ReplaceAll(k, "https://", "http://")
+		}
+		return k
+	}
+
+	var update bool
 	// Check if specified pools need to remove decommissioned pool.
 	for k := range specifiedPools {
 		pi, ok := rememberedPools[k]
+		if !ok {
+			pi, ok = rememberedPools[replaceScheme(k)]
+			if ok {
+				update = true // Looks like user is changing from http->https or https->http
+			}
+		}
 		if ok && pi.completed {
 			return false, fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)
 		}
@@ -320,6 +346,12 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 			continue
 		}
 		_, ok := specifiedPools[k]
+		if !ok {
+			_, ok = specifiedPools[replaceScheme(k)]
+			if ok {
+				update = true // Looks like user is changing from http->https or https->http
+			}
+		}
 		if !ok {
 			return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
 		}
@@ -329,6 +361,12 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 	if len(rememberedPools) == len(specifiedPools) {
 		for k, pi := range rememberedPools {
 			pos, ok := specifiedPools[k]
+			if !ok {
+				pos, ok = specifiedPools[replaceScheme(k)]
+				if ok {
+					update = true // Looks like user is changing from http->https or https->http
+				}
+			}
 			if !ok {
 				return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
 			}
@@ -338,7 +376,9 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 		}
 	}
 
-	update := len(rememberedPools) != len(specifiedPools)
+	if !update {
+		update = len(rememberedPools) != len(specifiedPools)
+	}
 	if update {
 		for k, pi := range rememberedPools {
 			if pi.decomStarted && !pi.completed {
@@ -466,21 +506,32 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 
 	// if no update is needed return right away.
 	if !update {
+		z.poolMeta = meta
+
 		// We are only supporting single pool decommission at this time
 		// so it makes sense to only resume single pools at any given
 		// time, in future meta.returnResumablePools() might take
 		// '-1' as argument to decommission multiple pools at a time
 		// but this is not a priority at the moment.
 		for _, pool := range meta.returnResumablePools(1) {
-			err := z.Decommission(ctx, pool.ID)
-			switch err {
-			case errDecommissionAlreadyRunning:
-				fallthrough
-			case nil:
-				go z.doDecommissionInRoutine(ctx, pool.ID)
+			idx := globalEndpoints.GetPoolIdx(pool.CmdLine)
+			if idx == -1 {
+				return fmt.Errorf("unexpected state present for decommission status pool(%s) not found", pool.CmdLine)
+			}
+			if globalEndpoints[idx].Endpoints[0].IsLocal {
+				go func(pool PoolStatus) {
+					switch err := z.Decommission(ctx, pool.ID); err {
+					case nil:
+						// we already started decommission
+					case errDecommissionAlreadyRunning:
+						// A previous decommission running found restart it.
+						z.doDecommissionInRoutine(ctx, idx)
+					default:
+						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+					}
+				}(pool)
 			}
 		}
-		z.poolMeta = meta
 
 		return nil
 	}
@@ -567,15 +618,18 @@ func (v versionsSorter) reverse() {
 }
 
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
-	var forwardTo string
-	// If we resume to the same bucket, forward to last known item.
-	rbucket, robject := z.poolMeta.ResumeBucketObject(idx)
-	if rbucket != "" && rbucket == bName {
-		forwardTo = robject
+	var wg sync.WaitGroup
+	wStr := env.Get("_MINIO_DECOMMISSION_WORKERS", strconv.Itoa(len(pool.sets)))
+	workerSize, err := strconv.Atoi(wStr)
+	if err != nil {
+		return err
 	}
 
+	parallelWorkers := make(chan struct{}, workerSize)
+
 	versioned := globalBucketVersioningSys.Enabled(bName)
 	for _, set := range pool.sets {
+		set := set
 		disks := set.getOnlineDisks()
 		if len(disks) == 0 {
 			logger.LogIf(GlobalContext, fmt.Errorf("no online disks found for set with endpoints %s",
@@ -584,6 +638,11 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		}
 
 		decommissionEntry := func(entry metaCacheEntry) {
+			defer func() {
+				<-parallelWorkers
+				wg.Done()
+			}()
+
 			if entry.isDir() {
 				return
 			}
@@ -597,15 +656,18 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
+			var decommissionedCount int
 			for _, version := range fivs.Versions {
 				// TODO: Skip transitioned objects for now.
 				if version.IsRemote() {
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bName, version.Name))
 					continue
 				}
 				// We will skip decommissioning delete markers
 				// with single version, its as good as there
 				// is no data associated with the object.
 				if version.Deleted && len(fivs.Versions) == 1 {
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bName, version.Name))
 					continue
 				}
 				if version.Deleted {
@@ -618,27 +680,24 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							MTime:             version.ModTime,
 							DeleteReplication: version.ReplicationState,
 						})
+					var failure bool
 					if err != nil {
 						logger.LogIf(ctx, err)
-						z.poolMetaMutex.Lock()
-						z.poolMeta.CountItem(idx, 0, true)
-						z.poolMetaMutex.Unlock()
-					} else {
-						set.DeleteObject(ctx,
-							bName,
-							version.Name,
-							ObjectOptions{
-								VersionID: version.VersionID,
-							})
-						z.poolMetaMutex.Lock()
-						z.poolMeta.CountItem(idx, 0, false)
-						z.poolMetaMutex.Unlock()
+						failure = true
 					}
+					z.poolMetaMutex.Lock()
+					z.poolMeta.CountItem(idx, 0, failure)
+					z.poolMetaMutex.Unlock()
+					if failure {
+						break // break out on first error
+					}
+					decommissionedCount++
 					continue
 				}
+
 				gr, err := set.GetObjectNInfo(ctx,
 					bName,
-					version.Name,
+					encodeDirObject(version.Name),
 					nil,
 					http.Header{},
 					noLock, // all mutations are blocked reads are safe without locks.
@@ -650,25 +709,32 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					z.poolMetaMutex.Lock()
 					z.poolMeta.CountItem(idx, version.Size, true)
 					z.poolMetaMutex.Unlock()
-					continue
+					break // break out on first error
 				}
+				var failure bool
 				// gr.Close() is ensured by decommissionObject().
 				if err = z.decommissionObject(ctx, bName, gr); err != nil {
 					logger.LogIf(ctx, err)
-					z.poolMetaMutex.Lock()
-					z.poolMeta.CountItem(idx, version.Size, true)
-					z.poolMetaMutex.Unlock()
-					continue
+					failure = true
 				}
+				z.poolMetaMutex.Lock()
+				z.poolMeta.CountItem(idx, version.Size, failure)
+				z.poolMetaMutex.Unlock()
+				if failure {
+					break // break out on first error
+				}
+				decommissionedCount++
+			}
+
+			// if all versions were decommissioned, then we can delete the object versions.
+			if decommissionedCount == len(fivs.Versions) {
 				set.DeleteObject(ctx,
 					bName,
-					version.Name,
+					entry.name,
 					ObjectOptions{
-						VersionID: version.VersionID,
-					})
-				z.poolMetaMutex.Lock()
-				z.poolMeta.CountItem(idx, version.Size, false)
-				z.poolMetaMutex.Unlock()
+						DeletePrefix: true, // use prefix delete to delete all versions at once.
+					},
+				)
 			}
 			z.poolMetaMutex.Lock()
 			z.poolMeta.TrackCurrentBucketObject(idx, bName, entry.name)
@@ -687,26 +753,35 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			bucket:    bName,
 		}
 
-		if err := listPathRaw(ctx, listPathRawOptions{
-			disks:          disks,
-			bucket:         bName,
-			recursive:      true,
-			forwardTo:      forwardTo,
-			minDisks:       len(disks) / 2, // to capture all quorum ratios
-			reportNotFound: false,
-			agreed:         decommissionEntry,
-			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-				entry, ok := entries.resolve(&resolver)
-				if ok {
-					decommissionEntry(*entry)
-				}
-			},
-			finished: nil,
-		}); err != nil {
-			// Decommissioning failed and won't continue
-			return err
-		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := listPathRaw(ctx, listPathRawOptions{
+				disks:          disks,
+				bucket:         bName,
+				recursive:      true,
+				forwardTo:      "",
+				minDisks:       len(disks) / 2, // to capture all quorum ratios
+				reportNotFound: false,
+				agreed: func(entry metaCacheEntry) {
+					parallelWorkers <- struct{}{}
+					wg.Add(1)
+					go decommissionEntry(entry)
+				},
+				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+					entry, ok := entries.resolve(&resolver)
+					if ok {
+						parallelWorkers <- struct{}{}
+						wg.Add(1)
+						go decommissionEntry(*entry)
+					}
+				},
+				finished: nil,
+			})
+			logger.LogIf(ctx, err)
+		}()
 	}
+	wg.Wait()
 	return nil
 }
 
@@ -749,13 +824,23 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
 		return
 	}
-	// Complete the decommission..
-	logger.LogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
+
+	z.poolMetaMutex.Lock()
+	failed := z.poolMeta.Pools[idx].Decommission.ItemsDecommissionFailed > 0
+	z.poolMetaMutex.Unlock()
+
+	if failed {
+		// Decommission failed indicate as such.
+		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+	} else {
+		// Complete the decommission..
+		logger.LogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
+	}
 }
 
 func (z *erasureServerPools) IsSuspended(idx int) bool {
-	z.poolMetaMutex.Lock()
-	defer z.poolMetaMutex.Unlock()
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
 	return z.poolMeta.IsSuspended(idx)
 }
 
@@ -801,12 +886,7 @@ func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpace
 	if idx+1 > len(z.serverPools) {
 		return pi, errInvalidArgument
 	}
-	info, errs := z.serverPools[idx].StorageInfo(context.Background())
-	for _, err := range errs {
-		if err != nil {
-			return pi, errInvalidArgument
-		}
-	}
+	info, _ := z.serverPools[idx].StorageInfo(context.Background())
 	info.Backend = z.BackendInfo()
 	for _, disk := range info.Disks {
 		if disk.Healing {
@@ -878,7 +958,7 @@ func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (e
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionCancel(idx) {
-		z.decommissionCancelers[idx]() // cancel any active thread.
+		defer z.decommissionCancelers[idx]() // cancel any active thread.
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -900,7 +980,7 @@ func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (e
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionFailed(idx) {
-		z.decommissionCancelers[idx]() // cancel any active thread.
+		defer z.decommissionCancelers[idx]() // cancel any active thread.
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -922,6 +1002,7 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionComplete(idx) {
+		defer z.decommissionCancelers[idx]() // cancel any active thread.
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -962,8 +1043,11 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		pathJoin(minioMetaBucket, minioConfigPrefix),
 		pathJoin(minioMetaBucket, bucketMetaPrefix),
 	} {
+		var bucketExists BucketExists
 		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
-			return err
+			if !errors.As(err, &bucketExists) {
+				return err
+			}
 		}
 	}
 

cmd/erasure-server-pool.go

@@ -27,12 +27,14 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -128,7 +130,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			if !configRetriableErrors(err) {
 				logger.Fatal(err, "Unable to initialize backend")
 			}
-			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
+			retry := time.Duration(r.Float64() * float64(5*time.Second))
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
+			time.Sleep(retry)
 			continue
 		}
 		break
@@ -528,7 +532,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) (StorageInfo, []er
 	return storageInfo, errs
 }
 
-func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
+func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
 	// Updates must be closed before we return.
 	defer close(updates)
 
@@ -573,7 +577,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 					}
 				}()
 				// Start scanner. Blocks until done.
-				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates)
+				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -817,6 +821,11 @@ func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, buc
 			// should be returned upwards.
 			return res.oi, res.zIdx, err
 		}
+		// When its a delete marker and versionID is empty
+		// we should simply return the error right away.
+		if res.oi.DeleteMarker && opts.VersionID == "" {
+			return res.oi, res.zIdx, err
+		}
 	}
 
 	object = decodeDirObject(object)
@@ -1150,6 +1159,12 @@ func maxKeysPlusOne(maxKeys int, addOne bool) int {
 func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
 
+	// Automatically remove the object/version is an expiry lifecycle rule can be applied
+	lc, _ := globalLifecycleSys.Get(bucket)
+
+	// Check if bucket is object locked.
+	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+
 	if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
@@ -1160,6 +1175,15 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		// to avoid the need for ListObjects().
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
+			if lc != nil {
+				action := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo, false)
+				switch action {
+				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+					fallthrough
+				case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+					return loi, nil
+				}
+			}
 			loi.Objects = append(loi.Objects, objInfo)
 			return loi, nil
 		}
@@ -1173,7 +1197,10 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		Marker:      marker,
 		InclDeleted: false,
 		AskDisks:    globalAPIConfig.getListQuorum(),
+		Lifecycle:   lc,
+		Retention:   rcfg,
 	}
+
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
@@ -1697,14 +1724,16 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 					}
 
 					path := baseDirFromPrefix(prefix)
-					if path == "" {
-						path = prefix
+					filterPrefix := strings.Trim(strings.TrimPrefix(prefix, path), slashSeparator)
+					if path == prefix {
+						filterPrefix = ""
 					}
 
 					lopts := listPathRawOptions{
 						disks:          disks,
 						bucket:         bucket,
 						path:           path,
+						filterPrefix:   filterPrefix,
 						recursive:      true,
 						forwardTo:      "",
 						minDisks:       1,
@@ -1757,14 +1786,16 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 	}
 
 	path := baseDirFromPrefix(prefix)
-	if path == "" {
-		path = prefix
+	filterPrefix := strings.Trim(strings.TrimPrefix(prefix, path), slashSeparator)
+	if path == prefix {
+		filterPrefix = ""
 	}
 
 	lopts := listPathRawOptions{
 		disks:          disks,
 		bucket:         bucket,
 		path:           path,
+		filterPrefix:   filterPrefix,
 		recursive:      true,
 		forwardTo:      "",
 		minDisks:       1,

cmd/erasure-sets.go

@@ -1099,7 +1099,8 @@ func (s *erasureSets) NewMultipartUpload(ctx context.Context, bucket, object str
 
 // Copies a part of an object from source hashedSet to destination hashedSet.
 func (s *erasureSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int,
-	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) {
+	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions,
+) (partInfo PartInfo, err error) {
 	destSet := s.getHashedSet(destObject)
 	auditObjectErasureSet(ctx, destObject, destSet)
 	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)

cmd/erasure.go

@@ -344,7 +344,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
-func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache) error {
+func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
 	if len(buckets) == 0 {
 		return nil
 	}
@@ -490,7 +490,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				// Calc usage
 				before := cache.Info.LastUpdate
 				var err error
-				cache, err = disk.NSScanner(ctx, cache, updates)
+				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
 				cache.Info.BloomFilter = nil
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {

cmd/fs-v1.go

@@ -235,7 +235,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 }
 
 // NSScanner returns data usage stats of the current FS deployment
-func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
+func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, _ madmin.HealScanMode) error {
 	defer close(updates)
 	// Load bucket totals
 	var totalCache dataUsageCache
@@ -396,7 +396,7 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 		}
 
 		return sizeSummary{totalSize: fi.Size(), versions: 1}, nil
-	})
+	}, 0)
 
 	return cache, err
 }
@@ -1362,13 +1362,15 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 
 // HealObject - no-op for fs. Valid only for Erasure.
 func (fs *FSObjects) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (
-	res madmin.HealResultItem, err error) {
+	res madmin.HealResultItem, err error,
+) {
 	return res, NotImplemented{}
 }
 
 // HealBucket - no-op for fs, Valid only for Erasure.
 func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem,
-	error) {
+	error,
+) {
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 

cmd/gateway-interface.go

@@ -19,14 +19,10 @@ package cmd
 
 import "github.com/minio/madmin-go"
 
-// GatewayMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
+// Gateway name backends
 const (
-	GatewayMinioSysTmp  = "minio.sys.tmp/"
-	AzureBackendGateway = "azure"
-	GCSBackendGateway   = "gcs"
-	HDFSBackendGateway  = "hdfs"
-	NASBackendGateway   = "nas"
-	S3BackendGateway    = "s3"
+	NASBackendGateway = "nas"
+	S3BackendGateway  = "s3"
 )
 
 // Gateway represents a gateway backend.

cmd/gateway-startup-msg_test.go

@@ -49,5 +49,5 @@ func TestPrintGatewayStartupMessage(t *testing.T) {
 	}
 
 	apiEndpoints := []string{"http://127.0.0.1:9000"}
-	printGatewayStartupMessage(apiEndpoints, "azure")
+	printGatewayStartupMessage(apiEndpoints, "s3")
 }

cmd/gateway-unsupported.go

@@ -48,7 +48,7 @@ func (a GatewayUnsupported) LocalStorageInfo(ctx context.Context) (StorageInfo,
 }
 
 // NSScanner - scanner is not implemented for gateway
-func (a GatewayUnsupported) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
+func (a GatewayUnsupported) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return NotImplemented{}
 }
@@ -211,7 +211,8 @@ func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix stri
 
 // CopyObject copies a blob from source container to destination container.
 func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
-	srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+	srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions,
+) (objInfo ObjectInfo, err error) {
 	return objInfo, NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure_test.go

@@ -1,273 +0,0 @@
-/*
- * MinIO Object Storage (c) 2021 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package azure
-
-import (
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"reflect"
-	"testing"
-
-	"github.com/Azure/azure-storage-blob-go/azblob"
-	minio "github.com/minio/minio/cmd"
-)
-
-func TestParseStorageEndpoint(t *testing.T) {
-	testCases := []struct {
-		host        string
-		accountName string
-		expectedURL string
-		expectedErr error
-	}{
-		{
-			"", "myaccount", "https://myaccount.blob.core.windows.net", nil,
-		},
-		{
-			"myaccount.blob.core.usgovcloudapi.net", "myaccount", "https://myaccount.blob.core.usgovcloudapi.net", nil,
-		},
-		{
-			"http://localhost:10000", "myaccount", "http://localhost:10000/myaccount", nil,
-		},
-	}
-	for i, testCase := range testCases {
-		endpointURL, err := parseStorageEndpoint(testCase.host, testCase.accountName)
-		if err != testCase.expectedErr {
-			t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
-		}
-		if endpointURL.String() != testCase.expectedURL {
-			t.Errorf("Test %d: Expected URL %s, got %s", i+1, testCase.expectedURL, endpointURL.String())
-		}
-	}
-}
-
-// Test canonical metadata.
-func TestS3MetaToAzureProperties(t *testing.T) {
-	headers := map[string]string{
-		"accept-encoding":          "gzip",
-		"content-encoding":         "gzip",
-		"cache-control":            "age: 3600",
-		"content-disposition":      "dummy",
-		"content-length":           "10",
-		"content-type":             "application/javascript",
-		"X-Amz-Meta-Hdr":           "value",
-		"X-Amz-Meta-X_test_key":    "value",
-		"X-Amz-Meta-X__test__key":  "value",
-		"X-Amz-Meta-X-Test__key":   "value",
-		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Amz-Meta-X-Amz-Matdesc": "{}",
-		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	// Only X-Amz-Meta- prefixed entries will be returned in
-	// Metadata (without the prefix!)
-	expectedHeaders := map[string]string{
-		"Hdr":              "value",
-		"X__test__key":     "value",
-		"X____test____key": "value",
-		"X_Test____key":    "value",
-		"X_Amz_Key":        "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X_Amz_Matdesc":    "{}",
-		"X_Amz_Iv":         "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	meta, _, err := s3MetaToAzureProperties(minio.GlobalContext, headers)
-	if err != nil {
-		t.Fatalf("Test failed, with %s", err)
-	}
-	if !reflect.DeepEqual(map[string]string(meta), expectedHeaders) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, meta)
-	}
-	headers = map[string]string{
-		"invalid--meta": "value",
-	}
-	_, _, err = s3MetaToAzureProperties(minio.GlobalContext, headers)
-	if err != nil {
-		if _, ok := err.(minio.UnsupportedMetadata); !ok {
-			t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
-		}
-	}
-
-	headers = map[string]string{
-		"content-md5": "Dce7bmCX61zvxzP5QmfelQ==",
-	}
-	_, props, err := s3MetaToAzureProperties(minio.GlobalContext, headers)
-	if err != nil {
-		t.Fatalf("Test failed, with %s", err)
-	}
-	if base64.StdEncoding.EncodeToString(props.ContentMD5) != headers["content-md5"] {
-		t.Fatalf("Test failed, expected %s, got %s", headers["content-md5"], props.ContentMD5)
-	}
-}
-
-func TestAzurePropertiesToS3Meta(t *testing.T) {
-	// Just one testcase. Adding more test cases does not add value to the testcase
-	// as azureToS3Metadata() just adds a prefix.
-	metadata := map[string]string{
-		"first_name":       "myname",
-		"x_test_key":       "value",
-		"x_test__key":      "value",
-		"x__test__key":     "value",
-		"x____test____key": "value",
-		"x_amz_key":        "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"x_amz_matdesc":    "{}",
-		"x_amz_iv":         "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	expectedMeta := map[string]string{
-		"X-Amz-Meta-First-Name":    "myname",
-		"X-Amz-Meta-X-Test-Key":    "value",
-		"X-Amz-Meta-X-Test_key":    "value",
-		"X-Amz-Meta-X_test_key":    "value",
-		"X-Amz-Meta-X__test__key":  "value",
-		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Amz-Meta-X-Amz-Matdesc": "{}",
-		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-		"Cache-Control":            "max-age: 3600",
-		"Content-Disposition":      "dummy",
-		"Content-Encoding":         "gzip",
-		"Content-Length":           "10",
-		"Content-MD5":              base64.StdEncoding.EncodeToString([]byte("base64-md5")),
-		"Content-Type":             "application/javascript",
-	}
-	actualMeta := azurePropertiesToS3Meta(metadata, azblob.BlobHTTPHeaders{
-		CacheControl:       "max-age: 3600",
-		ContentDisposition: "dummy",
-		ContentEncoding:    "gzip",
-		ContentMD5:         []byte("base64-md5"),
-		ContentType:        "application/javascript",
-	}, 10)
-	if !reflect.DeepEqual(actualMeta, expectedMeta) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, actualMeta)
-	}
-}
-
-// Add tests for azure to object error (top level).
-func TestAzureToObjectError(t *testing.T) {
-	testCases := []struct {
-		actualErr      error
-		expectedErr    error
-		bucket, object string
-	}{
-		{
-			nil, nil, "", "",
-		},
-		{
-			fmt.Errorf("Non azure error"),
-			fmt.Errorf("Non azure error"), "", "",
-		},
-	}
-	for i, testCase := range testCases {
-		if err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object); err != nil {
-			if err.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
-			}
-		} else {
-			if testCase.expectedErr != nil {
-				t.Errorf("Test %d expected an error but one was not produced", i+1)
-			}
-		}
-	}
-}
-
-// Add tests for azure to object error (internal).
-func TestAzureCodesToObjectError(t *testing.T) {
-	testCases := []struct {
-		originalErr       error
-		actualServiceCode string
-		actualStatusCode  int
-		expectedErr       error
-		bucket, object    string
-	}{
-		{
-			nil, "ContainerAlreadyExists", 0,
-			minio.BucketExists{Bucket: "bucket"},
-			"bucket", "",
-		},
-		{
-			nil, "InvalidResourceName", 0,
-			minio.BucketNameInvalid{Bucket: "bucket."},
-			"bucket.", "",
-		},
-		{
-			nil, "RequestBodyTooLarge", 0,
-			minio.PartTooBig{},
-			"", "",
-		},
-		{
-			nil, "InvalidMetadata", 0,
-			minio.UnsupportedMetadata{},
-			"", "",
-		},
-		{
-			nil, "", http.StatusNotFound,
-			minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-			"bucket", "object",
-		},
-		{
-			nil, "", http.StatusNotFound,
-			minio.BucketNotFound{Bucket: "bucket"},
-			"bucket", "",
-		},
-		{
-			nil, "", http.StatusBadRequest,
-			minio.BucketNameInvalid{Bucket: "bucket."},
-			"bucket.", "",
-		},
-		{
-			fmt.Errorf("unhandled azure error"), "", http.StatusForbidden,
-			fmt.Errorf("unhandled azure error"), "", "",
-		},
-	}
-	for i, testCase := range testCases {
-		if err := azureCodesToObjectError(testCase.originalErr, testCase.actualServiceCode, testCase.actualStatusCode, testCase.bucket, testCase.object); err != nil {
-			if err.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
-			}
-		} else {
-			if testCase.expectedErr != nil {
-				t.Errorf("Test %d expected an error but one was not produced", i+1)
-			}
-		}
-	}
-}
-
-func TestCheckAzureUploadID(t *testing.T) {
-	invalidUploadIDs := []string{
-		"123456789abcdefg",
-		"hello world",
-		"0x1234567890",
-		"1234567890abcdef1234567890abcdef",
-	}
-
-	for _, uploadID := range invalidUploadIDs {
-		if err := checkAzureUploadID(minio.GlobalContext, uploadID); err == nil {
-			t.Fatalf("%s: expected: <error>, got: <nil>", uploadID)
-		}
-	}
-
-	validUploadIDs := []string{
-		"1234567890abcdef",
-		"1122334455667788",
-	}
-
-	for _, uploadID := range validUploadIDs {
-		if err := checkAzureUploadID(minio.GlobalContext, uploadID); err != nil {
-			t.Fatalf("%s: expected: <nil>, got: %s", uploadID, err)
-		}
-	}
-}

cmd/gateway/gateway.go

@@ -22,17 +22,8 @@ import (
 	// NAS
 	_ "github.com/minio/minio/cmd/gateway/nas"
 
-	// Azure
-	_ "github.com/minio/minio/cmd/gateway/azure"
-
 	// S3
 	_ "github.com/minio/minio/cmd/gateway/s3"
-
-	// HDFS
-	_ "github.com/minio/minio/cmd/gateway/hdfs"
-
-	// GCS (use only if you must, GCS already supports S3 API)
-	_ "github.com/minio/minio/cmd/gateway/gcs"
 	// gateway functionality is frozen, no new gateways are being implemented
 	// or considered for upstream inclusion at this point in time. if needed
 	// please keep a fork of the project.

cmd/gateway/gcs/gateway-gcs_test.go

@@ -1,500 +0,0 @@
-/*
- * MinIO Object Storage (c) 2021 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package gcs
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"reflect"
-	"testing"
-	"time"
-
-	"cloud.google.com/go/storage"
-	"google.golang.org/api/googleapi"
-
-	miniogo "github.com/minio/minio-go/v7"
-	minio "github.com/minio/minio/cmd"
-)
-
-func TestToGCSPageToken(t *testing.T) {
-	testCases := []struct {
-		Name  string
-		Token string
-	}{
-		{
-			Name:  "A",
-			Token: "CgFB",
-		},
-		{
-			Name:  "AAAAAAAAAA",
-			Token: "CgpBQUFBQUFBQUFB",
-		},
-		{
-			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
-			Token: "CmRBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB",
-		},
-		{
-			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
-			Token: "CpEDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUE=",
-		},
-		{
-			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
-			Token: "CpIDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB",
-		},
-		{
-			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
-			Token: "CpMDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQQ==",
-		},
-		{
-			Name:  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
-			Token: "CvQDQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUE=",
-		},
-	}
-
-	for i, testCase := range testCases {
-		if toGCSPageToken(testCase.Name) != testCase.Token {
-			t.Errorf("Test %d: Expected %s, got %s", i+1, toGCSPageToken(testCase.Name), testCase.Token)
-		}
-	}
-}
-
-// TestIsValidGCSProjectIDFormat tests isValidGCSProjectIDFormat
-func TestValidGCSProjectIDFormat(t *testing.T) {
-	testCases := []struct {
-		ProjectID string
-		Valid     bool
-	}{
-		{"", false},
-		{"a", false},
-		{"Abc", false},
-		{"1bcd", false},
-		// 5 chars
-		{"abcdb", false},
-		// 6 chars
-		{"abcdbz", true},
-		// 30 chars
-		{"project-id-1-project-id-more-1", true},
-		// 31 chars
-		{"project-id-1-project-id-more-11", false},
-		{"storage.googleapis.com", false},
-		{"http://storage.googleapis.com", false},
-		{"http://localhost:9000", false},
-		{"project-id-1", true},
-		{"project-id-1988832", true},
-		{"projectid1414", true},
-	}
-
-	for i, testCase := range testCases {
-		valid := isValidGCSProjectIDFormat(testCase.ProjectID)
-		if valid != testCase.Valid {
-			t.Errorf("Test %d: Expected %v, got %v", i+1, valid, testCase.Valid)
-		}
-	}
-}
-
-// Test for isGCSMarker.
-func TestIsGCSMarker(t *testing.T) {
-	testCases := []struct {
-		marker   string
-		expected bool
-	}{
-		{
-			marker:   "{minio}gcs123",
-			expected: true,
-		},
-		{
-			marker:   "{mini_no}tgcs123",
-			expected: false,
-		},
-		{
-			marker:   "{minioagainnotgcs123",
-			expected: false,
-		},
-		{
-			marker:   "obj1",
-			expected: false,
-		},
-	}
-
-	for i, tc := range testCases {
-		if actual := isGCSMarker(tc.marker); actual != tc.expected {
-			t.Errorf("Test %d: marker is %s, expected %v but got %v",
-				i+1, tc.marker, tc.expected, actual)
-		}
-	}
-}
-
-// Test for gcsMultipartMetaName.
-func TestGCSMultipartMetaName(t *testing.T) {
-	uploadID := "a"
-	expected := path.Join(gcsMinioMultipartPathV1, uploadID, gcsMinioMultipartMeta)
-	got := gcsMultipartMetaName(uploadID)
-	if expected != got {
-		t.Errorf("expected: %s, got: %s", expected, got)
-	}
-}
-
-// Test for gcsMultipartDataName.
-func TestGCSMultipartDataName(t *testing.T) {
-	var (
-		uploadID   = "a"
-		etag       = "b"
-		partNumber = 1
-	)
-	expected := path.Join(gcsMinioMultipartPathV1, uploadID, fmt.Sprintf("%05d.%s", partNumber, etag))
-	got := gcsMultipartDataName(uploadID, partNumber, etag)
-	if expected != got {
-		t.Errorf("expected: %s, got: %s", expected, got)
-	}
-}
-
-func TestFromMinioClientListBucketResultToV2Info(t *testing.T) {
-	listBucketResult := miniogo.ListBucketResult{
-		IsTruncated:    false,
-		Marker:         "testMarker",
-		NextMarker:     "testMarker2",
-		CommonPrefixes: []miniogo.CommonPrefix{{Prefix: "one"}, {Prefix: "two"}},
-		Contents:       []miniogo.ObjectInfo{{Key: "testobj", ContentType: ""}},
-	}
-
-	listBucketV2Info := minio.ListObjectsV2Info{
-		Prefixes:              []string{"one", "two"},
-		Objects:               []minio.ObjectInfo{{Name: "testobj", Bucket: "testbucket", UserDefined: map[string]string{"Content-Type": ""}}},
-		IsTruncated:           false,
-		ContinuationToken:     "testMarker",
-		NextContinuationToken: "testMarker2",
-	}
-
-	if got := minio.FromMinioClientListBucketResultToV2Info("testbucket", listBucketResult); !reflect.DeepEqual(got, listBucketV2Info) {
-		t.Errorf("fromMinioClientListBucketResultToV2Info() = %v, want %v", got, listBucketV2Info)
-	}
-}
-
-// Test for gcsParseProjectID
-func TestGCSParseProjectID(t *testing.T) {
-	f, err := ioutil.TempFile("", "TestGCSParseProjectID-*")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-
-	contents := `
-{
-  "type": "service_account",
-  "project_id": "miniotesting"
-}
-`
-	f.WriteString(contents)
-	f.Close()
-	projectID, err := gcsParseProjectID(f.Name())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if projectID != "miniotesting" {
-		t.Errorf(`Expected projectID value to be "miniotesting"`)
-	}
-
-	if _, err = gcsParseProjectID("non-existent"); err == nil {
-		t.Errorf(`Expected to fail but succeeded reading "non-existent"`)
-	}
-
-	contents = `
-{
-  "type": "service_account",
-  "project_id": "miniotesting"
-},}
-`
-	f, err = ioutil.TempFile("", "TestGCSParseProjectID-*")
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer os.Remove(f.Name())
-	f.WriteString(contents)
-	f.Close()
-	if _, err := gcsParseProjectID(f.Name()); err == nil {
-		t.Errorf(`Expected to fail reading corrupted credentials file`)
-	}
-}
-
-func TestGCSToObjectError(t *testing.T) {
-	testCases := []struct {
-		params      []string
-		gcsErr      error
-		expectedErr error
-	}{
-		{
-			[]string{}, nil, nil,
-		},
-		{
-			[]string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"),
-		},
-		{
-			[]string{"bucket"},
-			fmt.Errorf("storage: bucket doesn't exist"),
-			minio.BucketNotFound{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			fmt.Errorf("storage: object doesn't exist"),
-			minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket", "object", "uploadID"},
-			fmt.Errorf("storage: object doesn't exist"),
-			minio.InvalidUploadID{
-				UploadID: "uploadID",
-			},
-		},
-		{
-			[]string{},
-			fmt.Errorf("Unknown error"),
-			fmt.Errorf("Unknown error"),
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Message: "No list of errors",
-			},
-			&googleapi.Error{
-				Message: "No list of errors",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason:  "conflict",
-					Message: "You already own this bucket. Please select another name.",
-				}},
-			},
-			minio.BucketAlreadyOwnedByYou{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason:  "conflict",
-					Message: "Sorry, that name is not available. Please try a different one.",
-				}},
-			},
-			minio.BucketAlreadyExists{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "conflict",
-				}},
-			},
-			minio.BucketNotEmpty{Bucket: "bucket"},
-		},
-		{
-			[]string{"bucket"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "notFound",
-				}},
-			},
-			minio.BucketNotFound{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "notFound",
-				}},
-			},
-			minio.ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "invalid",
-				}},
-			},
-			minio.BucketNameInvalid{
-				Bucket: "bucket",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "forbidden",
-				}},
-			},
-			minio.PrefixAccessDenied{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "keyInvalid",
-				}},
-			},
-			minio.PrefixAccessDenied{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "required",
-				}},
-			},
-			minio.PrefixAccessDenied{
-				Bucket: "bucket",
-				Object: "object",
-			},
-		},
-	}
-
-	for i, testCase := range testCases {
-		actualErr := gcsToObjectError(testCase.gcsErr, testCase.params...)
-		if actualErr != nil {
-			if actualErr.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
-			}
-		}
-	}
-}
-
-func TestS3MetaToGCSAttributes(t *testing.T) {
-	headers := map[string]string{
-		"accept-encoding":          "gzip",
-		"content-encoding":         "gzip",
-		"cache-control":            "age: 3600",
-		"content-disposition":      "dummy",
-		"content-type":             "application/javascript",
-		"Content-Language":         "en",
-		"X-Amz-Meta-Hdr":           "value",
-		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Amz-Meta-X-Amz-Matdesc": "{}",
-		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	// Only X-Amz-Meta- prefixed entries will be returned in
-	// Metadata (without the prefix!)
-	expectedHeaders := map[string]string{
-		"x-goog-meta-Hdr":           "value",
-		"x-goog-meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"x-goog-meta-X-Amz-Matdesc": "{}",
-		"x-goog-meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-
-	attrs := storage.ObjectAttrs{}
-	applyMetadataToGCSAttrs(headers, &attrs)
-
-	if !reflect.DeepEqual(attrs.Metadata, expectedHeaders) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, attrs.Metadata)
-	}
-
-	if attrs.CacheControl != headers["cache-control"] {
-		t.Fatalf("Test failed with Cache-Control mistmatch, expected %s, got %s", headers["cache-control"], attrs.CacheControl)
-	}
-	if attrs.ContentDisposition != headers["content-disposition"] {
-		t.Fatalf("Test failed with Content-Disposition mistmatch, expected %s, got %s", headers["content-disposition"], attrs.ContentDisposition)
-	}
-	if attrs.ContentEncoding != headers["content-encoding"] {
-		t.Fatalf("Test failed with Content-Encoding mistmatch, expected %s, got %s", headers["content-encoding"], attrs.ContentEncoding)
-	}
-	if attrs.ContentLanguage != headers["Content-Language"] {
-		t.Fatalf("Test failed with Content-Language mistmatch, expected %s, got %s", headers["Content-Language"], attrs.ContentLanguage)
-	}
-	if attrs.ContentType != headers["content-type"] {
-		t.Fatalf("Test failed with Content-Type mistmatch, expected %s, got %s", headers["content-type"], attrs.ContentType)
-	}
-}
-
-func TestGCSAttrsToObjectInfo(t *testing.T) {
-	metadata := map[string]string{
-		"x-goog-meta-Hdr":           "value",
-		"x-goog-meta-x_amz_key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"x-goog-meta-x-amz-matdesc": "{}",
-		"x-goog-meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	expectedMeta := map[string]string{
-		"X-Amz-Meta-Hdr":           "value",
-		"X-Amz-Meta-X_amz_key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Amz-Meta-X-Amz-Matdesc": "{}",
-		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-		"Cache-Control":            "max-age: 3600",
-		"Content-Disposition":      "dummy",
-		"Content-Encoding":         "gzip",
-		"Content-Language":         "en",
-		"Content-Type":             "application/javascript",
-	}
-
-	attrs := storage.ObjectAttrs{
-		Name:               "test-obj",
-		Bucket:             "test-bucket",
-		Updated:            time.Now(),
-		Size:               123,
-		CRC32C:             45312398,
-		CacheControl:       "max-age: 3600",
-		ContentDisposition: "dummy",
-		ContentEncoding:    "gzip",
-		ContentLanguage:    "en",
-		ContentType:        "application/javascript",
-		Metadata:           metadata,
-	}
-	expectedETag := minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C))
-
-	objInfo := fromGCSAttrsToObjectInfo(&attrs)
-	if !reflect.DeepEqual(objInfo.UserDefined, expectedMeta) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, objInfo.UserDefined)
-	}
-
-	if objInfo.Name != attrs.Name {
-		t.Fatalf("Test failed with Name mistmatch, expected %s, got %s", attrs.Name, objInfo.Name)
-	}
-	if objInfo.Bucket != attrs.Bucket {
-		t.Fatalf("Test failed with Bucket mistmatch, expected %s, got %s", attrs.Bucket, objInfo.Bucket)
-	}
-	if !objInfo.ModTime.Equal(attrs.Updated) {
-		t.Fatalf("Test failed with ModTime mistmatch, expected %s, got %s", attrs.Updated, objInfo.ModTime)
-	}
-	if objInfo.Size != attrs.Size {
-		t.Fatalf("Test failed with Size mistmatch, expected %d, got %d", attrs.Size, objInfo.Size)
-	}
-	if objInfo.ETag != expectedETag {
-		t.Fatalf("Test failed with ETag mistmatch, expected %s, got %s", expectedETag, objInfo.ETag)
-	}
-}

cmd/gateway/hdfs/gateway-hdfs-utils.go

@@ -1,67 +0,0 @@
-/*
- * MinIO Object Storage (c) 2021 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hdfs
-
-import (
-	"strings"
-
-	"github.com/minio/minio-go/v7/pkg/s3utils"
-	minio "github.com/minio/minio/cmd"
-)
-
-const (
-	// Minio meta bucket.
-	minioMetaBucket = ".minio.sys"
-
-	// Minio Tmp meta prefix.
-	minioMetaTmpBucket = minioMetaBucket + "/tmp"
-
-	// Minio reserved bucket name.
-	minioReservedBucket = "minio"
-)
-
-// Ignores all reserved bucket names or invalid bucket names.
-func isReservedOrInvalidBucket(bucketEntry string, strict bool) bool {
-	bucketEntry = strings.TrimSuffix(bucketEntry, minio.SlashSeparator)
-	if strict {
-		if err := s3utils.CheckValidBucketNameStrict(bucketEntry); err != nil {
-			return true
-		}
-	} else {
-		if err := s3utils.CheckValidBucketName(bucketEntry); err != nil {
-			return true
-		}
-	}
-	return isMinioMetaBucket(bucketEntry) || isMinioReservedBucket(bucketEntry)
-}
-
-// Returns true if input bucket is a reserved minio meta bucket '.minio.sys'.
-func isMinioMetaBucket(bucketName string) bool {
-	return bucketName == minioMetaBucket
-}
-
-// Returns true if input bucket is a reserved minio bucket 'minio'.
-func isMinioReservedBucket(bucketName string) bool {
-	return bucketName == minioReservedBucket
-}
-
-// byBucketName is a collection satisfying sort.Interface.
-type byBucketName []minio.BucketInfo
-
-func (d byBucketName) Len() int           { return len(d) }
-func (d byBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
-func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }

cmd/gateway/hdfs/gateway-hdfs.go

@@ -1,885 +0,0 @@
-/*
- * MinIO Object Storage (c) 2021 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hdfs
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"os"
-	"os/user"
-	"path"
-	"sort"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/colinmarc/hdfs/v2"
-	"github.com/colinmarc/hdfs/v2/hadoopconf"
-	krb "github.com/jcmturner/gokrb5/v8/client"
-	"github.com/jcmturner/gokrb5/v8/config"
-	"github.com/jcmturner/gokrb5/v8/credentials"
-	"github.com/jcmturner/gokrb5/v8/keytab"
-	"github.com/minio/cli"
-	"github.com/minio/madmin-go"
-	"github.com/minio/minio-go/v7/pkg/s3utils"
-	minio "github.com/minio/minio/cmd"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/env"
-	xnet "github.com/minio/pkg/net"
-)
-
-const (
-	hdfsSeparator = minio.SlashSeparator
-)
-
-func init() {
-	const hdfsGatewayTemplate = `NAME:
-  {{.HelpName}} - {{.Usage}}
-
-USAGE:
-  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} HDFS-NAMENODE [HDFS-NAMENODE...]
-{{if .VisibleFlags}}
-FLAGS:
-  {{range .VisibleFlags}}{{.}}
-  {{end}}{{end}}
-HDFS-NAMENODE:
-  HDFS namenode URI
-
-EXAMPLES:
-  1. Start minio gateway server for HDFS backend
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
-     {{.Prompt}} {{.HelpName}} hdfs://namenode:8200
-
-  2. Start minio gateway server for HDFS with edge caching enabled
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
-     {{.Prompt}} {{.HelpName}} hdfs://namenode:8200
-`
-
-	minio.RegisterGatewayCommand(cli.Command{
-		Name:               minio.HDFSBackendGateway,
-		Usage:              "Hadoop Distributed File System (HDFS)",
-		Action:             hdfsGatewayMain,
-		CustomHelpTemplate: hdfsGatewayTemplate,
-		HideHelpCommand:    true,
-	})
-}
-
-// Handler for 'minio gateway hdfs' command line.
-func hdfsGatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	if ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, minio.HDFSBackendGateway, 1)
-	}
-
-	minio.StartGateway(ctx, &HDFS{args: ctx.Args()})
-}
-
-// HDFS implements Gateway.
-type HDFS struct {
-	args []string
-}
-
-// Name implements Gateway interface.
-func (g *HDFS) Name() string {
-	return minio.HDFSBackendGateway
-}
-
-func getKerberosClient() (*krb.Client, error) {
-	cfg, err := config.Load(env.Get("KRB5_CONFIG", "/etc/krb5.conf"))
-	if err != nil {
-		return nil, err
-	}
-
-	u, err := user.Current()
-	if err != nil {
-		return nil, err
-	}
-
-	keytabPath := env.Get("KRB5KEYTAB", "")
-	if keytabPath != "" {
-		kt, err := keytab.Load(keytabPath)
-		if err != nil {
-			return nil, err
-		}
-
-		username := env.Get("KRB5USERNAME", "")
-		realm := env.Get("KRB5REALM", "")
-		if username == "" || realm == "" {
-			return nil, errors.New("empty KRB5USERNAME or KRB5REALM")
-		}
-
-		return krb.NewWithKeytab(username, realm, kt, cfg), nil
-	}
-
-	// Determine the ccache location from the environment, falling back to the default location.
-	ccachePath := env.Get("KRB5CCNAME", fmt.Sprintf("/tmp/krb5cc_%s", u.Uid))
-	if strings.Contains(ccachePath, ":") {
-		if strings.HasPrefix(ccachePath, "FILE:") {
-			ccachePath = strings.TrimPrefix(ccachePath, "FILE:")
-		} else {
-			return nil, fmt.Errorf("unable to use kerberos ccache: %s", ccachePath)
-		}
-	}
-
-	ccache, err := credentials.LoadCCache(ccachePath)
-	if err != nil {
-		return nil, err
-	}
-
-	return krb.NewFromCCache(ccache, cfg)
-}
-
-// NewGatewayLayer returns hdfs gatewaylayer.
-func (g *HDFS) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, error) {
-	dialFunc := (&net.Dialer{
-		Timeout:   30 * time.Second,
-		KeepAlive: 30 * time.Second,
-		DualStack: true,
-	}).DialContext
-
-	hconfig, err := hadoopconf.LoadFromEnvironment()
-	if err != nil {
-		return nil, err
-	}
-
-	opts := hdfs.ClientOptionsFromConf(hconfig)
-	opts.NamenodeDialFunc = dialFunc
-	opts.DatanodeDialFunc = dialFunc
-
-	// Not addresses found, load it from command line.
-	var commonPath string
-	if len(opts.Addresses) == 0 {
-		var addresses []string
-		for _, s := range g.args {
-			u, err := xnet.ParseURL(s)
-			if err != nil {
-				return nil, err
-			}
-			if u.Scheme != "hdfs" {
-				return nil, fmt.Errorf("unsupported scheme %s, only supports hdfs://", u)
-			}
-			if commonPath != "" && commonPath != u.Path {
-				return nil, fmt.Errorf("all namenode paths should be same %s", g.args)
-			}
-			if commonPath == "" {
-				commonPath = u.Path
-			}
-			addresses = append(addresses, u.Host)
-		}
-		opts.Addresses = addresses
-	}
-
-	u, err := user.Current()
-	if err != nil {
-		return nil, fmt.Errorf("unable to lookup local user: %s", err)
-	}
-
-	if opts.KerberosClient != nil {
-		opts.KerberosClient, err = getKerberosClient()
-		if err != nil {
-			return nil, fmt.Errorf("unable to initialize kerberos client: %s", err)
-		}
-	} else {
-		opts.User = env.Get("HADOOP_USER_NAME", u.Username)
-	}
-
-	clnt, err := hdfs.NewClient(opts)
-	if err != nil {
-		return nil, fmt.Errorf("unable to initialize hdfsClient: %v", err)
-	}
-
-	if err = clnt.MkdirAll(minio.PathJoin(commonPath, hdfsSeparator, minioMetaTmpBucket), os.FileMode(0o755)); err != nil {
-		return nil, err
-	}
-
-	return &hdfsObjects{clnt: clnt, subPath: commonPath, listPool: minio.NewTreeWalkPool(time.Minute * 30)}, nil
-}
-
-func (n *hdfsObjects) Shutdown(ctx context.Context) error {
-	return n.clnt.Close()
-}
-
-func (n *hdfsObjects) LocalStorageInfo(ctx context.Context) (si minio.StorageInfo, errs []error) {
-	return n.StorageInfo(ctx)
-}
-
-func (n *hdfsObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo, errs []error) {
-	fsInfo, err := n.clnt.StatFs()
-	if err != nil {
-		return minio.StorageInfo{}, []error{err}
-	}
-	si.Disks = []madmin.Disk{{
-		UsedSpace: fsInfo.Used,
-	}}
-	si.Backend.Type = madmin.Gateway
-	si.Backend.GatewayOnline = true
-	return si, nil
-}
-
-// hdfsObjects implements gateway for Minio and S3 compatible object storage servers.
-type hdfsObjects struct {
-	minio.GatewayUnsupported
-	clnt     *hdfs.Client
-	subPath  string
-	listPool *minio.TreeWalkPool
-}
-
-func hdfsToObjectErr(ctx context.Context, err error, params ...string) error {
-	if err == nil {
-		return nil
-	}
-	bucket := ""
-	object := ""
-	uploadID := ""
-	switch len(params) {
-	case 3:
-		uploadID = params[2]
-		fallthrough
-	case 2:
-		object = params[1]
-		fallthrough
-	case 1:
-		bucket = params[0]
-	}
-
-	switch {
-	case os.IsNotExist(err):
-		if uploadID != "" {
-			return minio.InvalidUploadID{
-				UploadID: uploadID,
-			}
-		}
-		if object != "" {
-			return minio.ObjectNotFound{Bucket: bucket, Object: object}
-		}
-		return minio.BucketNotFound{Bucket: bucket}
-	case os.IsExist(err):
-		if object != "" {
-			return minio.PrefixAccessDenied{Bucket: bucket, Object: object}
-		}
-		return minio.BucketAlreadyOwnedByYou{Bucket: bucket}
-	case errors.Is(err, syscall.ENOTEMPTY):
-		if object != "" {
-			return minio.PrefixAccessDenied{Bucket: bucket, Object: object}
-		}
-		return minio.BucketNotEmpty{Bucket: bucket}
-	default:
-		logger.LogIf(ctx, err)
-		return err
-	}
-}
-
-// hdfsIsValidBucketName verifies whether a bucket name is valid.
-func hdfsIsValidBucketName(bucket string) bool {
-	return s3utils.CheckValidBucketNameStrict(bucket) == nil
-}
-
-func (n *hdfsObjects) hdfsPathJoin(args ...string) string {
-	return minio.PathJoin(append([]string{n.subPath, hdfsSeparator}, args...)...)
-}
-
-func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
-	if !hdfsIsValidBucketName(bucket) {
-		return minio.BucketNameInvalid{Bucket: bucket}
-	}
-	if opts.Force {
-		return hdfsToObjectErr(ctx, n.clnt.RemoveAll(n.hdfsPathJoin(bucket)), bucket)
-	}
-	return hdfsToObjectErr(ctx, n.clnt.Remove(n.hdfsPathJoin(bucket)), bucket)
-}
-
-func (n *hdfsObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
-	if opts.LockEnabled || opts.VersioningEnabled {
-		return minio.NotImplemented{}
-	}
-
-	if !hdfsIsValidBucketName(bucket) {
-		return minio.BucketNameInvalid{Bucket: bucket}
-	}
-	return hdfsToObjectErr(ctx, n.clnt.Mkdir(n.hdfsPathJoin(bucket), os.FileMode(0o755)), bucket)
-}
-
-func (n *hdfsObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
-	fi, err := n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return bi, hdfsToObjectErr(ctx, err, bucket)
-	}
-	// As hdfs.Stat() doesn't carry anything other than ModTime(), use ModTime() as CreatedTime.
-	return minio.BucketInfo{
-		Name:    bucket,
-		Created: fi.ModTime(),
-	}, nil
-}
-
-func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
-	entries, err := n.clnt.ReadDir(n.hdfsPathJoin())
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil, hdfsToObjectErr(ctx, err)
-	}
-
-	for _, entry := range entries {
-		// Ignore all reserved bucket names and invalid bucket names.
-		if isReservedOrInvalidBucket(entry.Name(), false) {
-			continue
-		}
-		buckets = append(buckets, minio.BucketInfo{
-			Name: entry.Name(),
-			// As hdfs.Stat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
-			Created: entry.ModTime(),
-		})
-	}
-
-	// Sort bucket infos by bucket name.
-	sort.Sort(byBucketName(buckets))
-	return buckets, nil
-}
-
-func (n *hdfsObjects) isLeafDir(bucket, leafPath string) bool {
-	return n.isObjectDir(context.Background(), bucket, leafPath)
-}
-
-func (n *hdfsObjects) isLeaf(bucket, leafPath string) bool {
-	return !strings.HasSuffix(leafPath, hdfsSeparator)
-}
-
-func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
-	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
-		f, err := n.clnt.Open(n.hdfsPathJoin(bucket, prefixDir))
-		if err != nil {
-			if os.IsNotExist(err) {
-				err = nil
-			}
-			logger.LogIf(minio.GlobalContext, err)
-			return
-		}
-		defer f.Close()
-		fis, err := f.Readdir(0)
-		if err != nil {
-			logger.LogIf(minio.GlobalContext, err)
-			return
-		}
-		if len(fis) == 0 {
-			return true, nil, false
-		}
-		for _, fi := range fis {
-			if fi.IsDir() {
-				entries = append(entries, fi.Name()+hdfsSeparator)
-			} else {
-				entries = append(entries, fi.Name())
-			}
-		}
-		entries, delayIsLeaf = minio.FilterListEntries(bucket, prefixDir, entries, prefixEntry, n.isLeaf)
-		return false, entries, delayIsLeaf
-	}
-
-	// Return list factory instance.
-	return listDir
-}
-
-// ListObjects lists all blobs in HDFS bucket filtered by prefix.
-func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
-	var mutex sync.Mutex
-	fileInfos := make(map[string]os.FileInfo)
-	targetPath := n.hdfsPathJoin(bucket, prefix)
-
-	var targetFileInfo os.FileInfo
-
-	if targetFileInfo, err = n.populateDirectoryListing(targetPath, fileInfos); err != nil {
-		return loi, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	// If the user is trying to list a single file, bypass the entire directory-walking code below
-	// and just return the single file's information.
-	if !targetFileInfo.IsDir() {
-		return minio.ListObjectsInfo{
-			IsTruncated: false,
-			NextMarker:  "",
-			Objects: []minio.ObjectInfo{
-				fileInfoToObjectInfo(bucket, prefix, targetFileInfo),
-			},
-			Prefixes: []string{},
-		}, nil
-	}
-
-	getObjectInfo := func(ctx context.Context, bucket, entry string) (minio.ObjectInfo, error) {
-		mutex.Lock()
-		defer mutex.Unlock()
-
-		filePath := path.Clean(n.hdfsPathJoin(bucket, entry))
-		fi, ok := fileInfos[filePath]
-
-		// If the file info is not known, this may be a recursive listing and filePath is a
-		// child of a sub-directory. In this case, obtain that sub-directory's listing.
-		if !ok {
-			parentPath := path.Dir(filePath)
-
-			if _, err := n.populateDirectoryListing(parentPath, fileInfos); err != nil {
-				return minio.ObjectInfo{}, hdfsToObjectErr(ctx, err, bucket)
-			}
-
-			fi, ok = fileInfos[filePath]
-
-			if !ok {
-				err = fmt.Errorf("could not get FileInfo for path '%s'", filePath)
-				return minio.ObjectInfo{}, hdfsToObjectErr(ctx, err, bucket, entry)
-			}
-		}
-
-		objectInfo := fileInfoToObjectInfo(bucket, entry, fi)
-
-		delete(fileInfos, filePath)
-
-		return objectInfo, nil
-	}
-
-	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), n.isLeaf, n.isLeafDir, getObjectInfo, getObjectInfo)
-}
-
-func fileInfoToObjectInfo(bucket string, entry string, fi os.FileInfo) minio.ObjectInfo {
-	return minio.ObjectInfo{
-		Bucket:  bucket,
-		Name:    entry,
-		ModTime: fi.ModTime(),
-		Size:    fi.Size(),
-		IsDir:   fi.IsDir(),
-		AccTime: fi.(*hdfs.FileInfo).AccessTime(),
-	}
-}
-
-// Lists a path's direct, first-level entries and populates them in the `fileInfos` cache which maps
-// a path entry to an `os.FileInfo`. It also saves the listed path's `os.FileInfo` in the cache.
-func (n *hdfsObjects) populateDirectoryListing(filePath string, fileInfos map[string]os.FileInfo) (os.FileInfo, error) {
-	dirReader, err := n.clnt.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-
-	dirStat := dirReader.Stat()
-	key := path.Clean(filePath)
-
-	if !dirStat.IsDir() {
-		return dirStat, nil
-	}
-
-	fileInfos[key] = dirStat
-	infos, err := dirReader.Readdir(0)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, fileInfo := range infos {
-		filePath := minio.PathJoin(filePath, fileInfo.Name())
-		fileInfos[filePath] = fileInfo
-	}
-
-	return dirStat, nil
-}
-
-// deleteObject deletes a file path if its empty. If it's successfully deleted,
-// it will recursively move up the tree, deleting empty parent directories
-// until it finds one with files in it. Returns nil for a non-empty directory.
-func (n *hdfsObjects) deleteObject(basePath, deletePath string) error {
-	if basePath == deletePath {
-		return nil
-	}
-
-	// Attempt to remove path.
-	if err := n.clnt.Remove(deletePath); err != nil {
-		if errors.Is(err, syscall.ENOTEMPTY) {
-			// Ignore errors if the directory is not empty. The server relies on
-			// this functionality, and sometimes uses recursion that should not
-			// error on parent directories.
-			return nil
-		}
-		return err
-	}
-
-	// Trailing slash is removed when found to ensure
-	// slashpath.Dir() to work as intended.
-	deletePath = strings.TrimSuffix(deletePath, hdfsSeparator)
-	deletePath = path.Dir(deletePath)
-
-	// Delete parent directory. Errors for parent directories shouldn't trickle down.
-	n.deleteObject(basePath, deletePath)
-
-	return nil
-}
-
-// ListObjectsV2 lists all blobs in HDFS bucket filtered by prefix
-func (n *hdfsObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
-	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner is not supported and unused.
-	marker := continuationToken
-	if marker == "" {
-		marker = startAfter
-	}
-	resultV1, err := n.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
-	if err != nil {
-		return loi, err
-	}
-	return minio.ListObjectsV2Info{
-		Objects:               resultV1.Objects,
-		Prefixes:              resultV1.Prefixes,
-		ContinuationToken:     continuationToken,
-		NextContinuationToken: resultV1.NextMarker,
-		IsTruncated:           resultV1.IsTruncated,
-	}, nil
-}
-
-func (n *hdfsObjects) DeleteObject(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
-	err := hdfsToObjectErr(ctx, n.deleteObject(n.hdfsPathJoin(bucket), n.hdfsPathJoin(bucket, object)), bucket, object)
-	return minio.ObjectInfo{
-		Bucket: bucket,
-		Name:   object,
-	}, err
-}
-
-func (n *hdfsObjects) DeleteObjects(ctx context.Context, bucket string, objects []minio.ObjectToDelete, opts minio.ObjectOptions) ([]minio.DeletedObject, []error) {
-	errs := make([]error, len(objects))
-	dobjects := make([]minio.DeletedObject, len(objects))
-	for idx, object := range objects {
-		_, errs[idx] = n.DeleteObject(ctx, bucket, object.ObjectName, opts)
-		if errs[idx] == nil {
-			dobjects[idx] = minio.DeletedObject{
-				ObjectName: object.ObjectName,
-			}
-		}
-	}
-	return dobjects, errs
-}
-
-func (n *hdfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
-	objInfo, err := n.GetObjectInfo(ctx, bucket, object, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	var startOffset, length int64
-	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
-	if err != nil {
-		return nil, err
-	}
-
-	pr, pw := io.Pipe()
-	go func() {
-		nerr := n.getObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
-		pw.CloseWithError(nerr)
-	}()
-
-	// Setup cleanup function to cause the above go-routine to
-	// exit in case of partial read
-	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
-}
-
-func (n *hdfsObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.ObjectInfo, error) {
-	cpSrcDstSame := minio.IsStringEqual(n.hdfsPathJoin(srcBucket, srcObject), n.hdfsPathJoin(dstBucket, dstObject))
-	if cpSrcDstSame {
-		return n.GetObjectInfo(ctx, srcBucket, srcObject, minio.ObjectOptions{})
-	}
-
-	return n.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, minio.ObjectOptions{
-		ServerSideEncryption: dstOpts.ServerSideEncryption,
-		UserDefined:          srcInfo.UserDefined,
-	})
-}
-
-func (n *hdfsObjects) getObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
-	if _, err := n.clnt.Stat(n.hdfsPathJoin(bucket)); err != nil {
-		return hdfsToObjectErr(ctx, err, bucket)
-	}
-	rd, err := n.clnt.Open(n.hdfsPathJoin(bucket, key))
-	if err != nil {
-		return hdfsToObjectErr(ctx, err, bucket, key)
-	}
-	defer rd.Close()
-	_, err = io.Copy(writer, io.NewSectionReader(rd, startOffset, length))
-	if err == io.ErrClosedPipe {
-		// hdfs library doesn't send EOF correctly, so io.Copy attempts
-		// to write which returns io.ErrClosedPipe - just ignore
-		// this for now.
-		err = nil
-	}
-	return hdfsToObjectErr(ctx, err, bucket, key)
-}
-
-func (n *hdfsObjects) isObjectDir(ctx context.Context, bucket, object string) bool {
-	f, err := n.clnt.Open(n.hdfsPathJoin(bucket, object))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-		logger.LogIf(ctx, err)
-		return false
-	}
-	defer f.Close()
-	fis, err := f.Readdir(1)
-	if err != nil && err != io.EOF {
-		logger.LogIf(ctx, err)
-		return false
-	}
-	// Readdir returns an io.EOF when len(fis) == 0.
-	return len(fis) == 0
-}
-
-// GetObjectInfo reads object info and replies back ObjectInfo.
-func (n *hdfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return objInfo, hdfsToObjectErr(ctx, err, bucket)
-	}
-	if strings.HasSuffix(object, hdfsSeparator) && !n.isObjectDir(ctx, bucket, object) {
-		return objInfo, hdfsToObjectErr(ctx, os.ErrNotExist, bucket, object)
-	}
-
-	fi, err := n.clnt.Stat(n.hdfsPathJoin(bucket, object))
-	if err != nil {
-		return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-	}
-	return minio.ObjectInfo{
-		Bucket:  bucket,
-		Name:    object,
-		ModTime: fi.ModTime(),
-		Size:    fi.Size(),
-		IsDir:   fi.IsDir(),
-		AccTime: fi.(*hdfs.FileInfo).AccessTime(),
-	}, nil
-}
-
-func (n *hdfsObjects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return objInfo, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	name := n.hdfsPathJoin(bucket, object)
-
-	// If its a directory create a prefix {
-	if strings.HasSuffix(object, hdfsSeparator) && r.Size() == 0 {
-		if err = n.clnt.MkdirAll(name, os.FileMode(0o755)); err != nil {
-			n.deleteObject(n.hdfsPathJoin(bucket), name)
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-	} else {
-		tmpname := n.hdfsPathJoin(minioMetaTmpBucket, minio.MustGetUUID())
-		var w *hdfs.FileWriter
-		w, err = n.clnt.Create(tmpname)
-		if err != nil {
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-		defer n.deleteObject(n.hdfsPathJoin(minioMetaTmpBucket), tmpname)
-		if _, err = io.Copy(w, r); err != nil {
-			w.Close()
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-		dir := path.Dir(name)
-		if dir != "" {
-			if err = n.clnt.MkdirAll(dir, os.FileMode(0o755)); err != nil {
-				w.Close()
-				n.deleteObject(n.hdfsPathJoin(bucket), dir)
-				return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-			}
-		}
-		w.Close()
-		if err = n.clnt.Rename(tmpname, name); err != nil {
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-	}
-	fi, err := n.clnt.Stat(name)
-	if err != nil {
-		return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-	}
-	return minio.ObjectInfo{
-		Bucket:  bucket,
-		Name:    object,
-		ETag:    r.MD5CurrentHexString(),
-		ModTime: fi.ModTime(),
-		Size:    fi.Size(),
-		IsDir:   fi.IsDir(),
-		AccTime: fi.(*hdfs.FileInfo).AccessTime(),
-	}, nil
-}
-
-func (n *hdfsObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (uploadID string, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return uploadID, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	uploadID = minio.MustGetUUID()
-	if err = n.clnt.CreateEmptyFile(n.hdfsPathJoin(minioMetaTmpBucket, uploadID)); err != nil {
-		return uploadID, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	return uploadID, nil
-}
-
-func (n *hdfsObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return lmi, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	// It's decided not to support List Multipart Uploads, hence returning empty result.
-	return lmi, nil
-}
-
-func (n *hdfsObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) (err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(minioMetaTmpBucket, uploadID))
-	if err != nil {
-		return hdfsToObjectErr(ctx, err, bucket, object, uploadID)
-	}
-	return nil
-}
-
-// GetMultipartInfo returns multipart info of the uploadId of the object
-func (n *hdfsObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts minio.ObjectOptions) (result minio.MultipartInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return result, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	if err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
-		return result, err
-	}
-
-	result.Bucket = bucket
-	result.Object = object
-	result.UploadID = uploadID
-	return result, nil
-}
-
-func (n *hdfsObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (result minio.ListPartsInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return result, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	if err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
-		return result, err
-	}
-
-	// It's decided not to support List parts, hence returning empty result.
-	return result, nil
-}
-
-func (n *hdfsObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,
-	startOffset int64, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.PartInfo, error) {
-	return n.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
-}
-
-func (n *hdfsObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return info, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	var w *hdfs.FileWriter
-	w, err = n.clnt.Append(n.hdfsPathJoin(minioMetaTmpBucket, uploadID))
-	if err != nil {
-		return info, hdfsToObjectErr(ctx, err, bucket, object, uploadID)
-	}
-	defer w.Close()
-	_, err = io.Copy(w, r.Reader)
-	if err != nil {
-		return info, hdfsToObjectErr(ctx, err, bucket, object, uploadID)
-	}
-
-	info.PartNumber = partID
-	info.ETag = r.MD5CurrentHexString()
-	info.LastModified = minio.UTCNow()
-	info.Size = r.Reader.Size()
-
-	return info, nil
-}
-
-func (n *hdfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []minio.CompletePart, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return objInfo, hdfsToObjectErr(ctx, err, bucket)
-	}
-
-	if err = n.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
-		return objInfo, err
-	}
-
-	name := n.hdfsPathJoin(bucket, object)
-	dir := path.Dir(name)
-	if dir != "" {
-		if err = n.clnt.MkdirAll(dir, os.FileMode(0o755)); err != nil {
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-	}
-
-	err = n.clnt.Rename(n.hdfsPathJoin(minioMetaTmpBucket, uploadID), name)
-	// Object already exists is an error on HDFS
-	// remove it and then create it again.
-	if os.IsExist(err) {
-		if err = n.clnt.Remove(name); err != nil {
-			if dir != "" {
-				n.deleteObject(n.hdfsPathJoin(bucket), dir)
-			}
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-		if err = n.clnt.Rename(n.hdfsPathJoin(minioMetaTmpBucket, uploadID), name); err != nil {
-			if dir != "" {
-				n.deleteObject(n.hdfsPathJoin(bucket), dir)
-			}
-			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-		}
-	}
-	fi, err := n.clnt.Stat(name)
-	if err != nil {
-		return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
-	}
-
-	// Calculate s3 compatible md5sum for complete multipart.
-	s3MD5 := minio.ComputeCompleteMultipartMD5(parts)
-
-	return minio.ObjectInfo{
-		Bucket:  bucket,
-		Name:    object,
-		ETag:    s3MD5,
-		ModTime: fi.ModTime(),
-		Size:    fi.Size(),
-		IsDir:   fi.IsDir(),
-		AccTime: fi.(*hdfs.FileInfo).AccessTime(),
-	}, nil
-}
-
-func (n *hdfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts minio.ObjectOptions) (err error) {
-	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
-	if err != nil {
-		return hdfsToObjectErr(ctx, err, bucket)
-	}
-	return hdfsToObjectErr(ctx, n.clnt.Remove(n.hdfsPathJoin(minioMetaTmpBucket, uploadID)), bucket, object, uploadID)
-}

cmd/gateway/s3/gateway-s3-sse.go

@@ -549,7 +549,8 @@ func (l *s3EncObjects) PutObjectPart(ctx context.Context, bucket string, object
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *s3EncObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions,
+) (p minio.PartInfo, err error) {
 	return l.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
 }
 

cmd/gateway/s3/gateway-s3.go

@@ -640,7 +640,8 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions,
+) (p minio.PartInfo, err error) {
 	if srcOpts.CheckPrecondFn != nil && srcOpts.CheckPrecondFn(srcInfo) {
 		return minio.PartInfo{}, minio.PreConditionFailed{}
 	}

cmd/global-heal.go

@@ -44,8 +44,7 @@ func newBgHealSequence() *healSequence {
 
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
-		Remove:   healDeleteDangling,
-		ScanMode: globalHealConfig.ScanMode(),
+		Remove: healDeleteDangling,
 	}
 
 	return &healSequence{
@@ -165,7 +164,7 @@ func mustGetHealSequence(ctx context.Context) *healSequence {
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
 	bgSeq := mustGetHealSequence(ctx)
-	scanMode := globalHealConfig.ScanMode()
+	scanMode := madmin.HealNormalScan
 
 	// Make sure to copy since `buckets slice`
 	// is modified in place by tracker.

cmd/globals.go

@@ -102,7 +102,7 @@ const (
 	GlobalStaleUploadsCleanupInterval = time.Hour * 6 // 6 hrs.
 
 	// Refresh interval to update in-memory iam config cache.
-	globalRefreshIAMInterval = 5 * time.Minute
+	globalRefreshIAMInterval = 10 * time.Minute
 
 	// Limit of location constraint XML for unauthenticated PUT bucket operations.
 	maxLocationConstraintSize = 3 * humanize.MiByte
@@ -139,7 +139,7 @@ var (
 	// Indicates if the running minio is in gateway mode.
 	globalIsGateway = false
 
-	// Name of gateway server, e.g S3, GCS, Azure, etc
+	// Name of gateway server, e.g S3, NAS etc
 	globalGatewayName = ""
 
 	// This flag is set to 'true' by default
@@ -190,7 +190,7 @@ var (
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
 	// healthcheck readiness deadlines and cors settings.
-	globalAPIConfig = apiConfig{listQuorum: 3}
+	globalAPIConfig = apiConfig{listQuorum: "strict"}
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -290,9 +290,6 @@ var (
 	// Some standard content-types which we strictly dis-allow for compression.
 	standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
 
-	// Authorization validators list.
-	globalOpenIDValidators *openid.Validators
-
 	// OPA policy system.
 	globalPolicyOPA *opa.Opa
 
@@ -347,8 +344,10 @@ var (
 	globalRootDiskThreshold uint64
 
 	// Used for collecting stats for netperf
-	globalNetPerfMinDuration = time.Second * 10
-	globalNetPerfRX          netPerfRX
+	globalNetPerfMinDuration     = time.Second * 10
+	globalNetPerfRX              netPerfRX
+	globalObjectPerfBucket       = "minio-perf-test-tmp-bucket"
+	globalObjectPerfUserMetadata = "X-Amz-Meta-Minio-Object-Perf" // Clients can set this to bypass S3 API service freeze. Used by object pref tests.
 
 	// Add new variable global values here.
 )

cmd/handler-api.go

@@ -38,7 +38,7 @@ type apiConfig struct {
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
 	clusterDeadline  time.Duration
-	listQuorum       int
+	listQuorum       string
 	corsAllowOrigins []string
 	// total drives per erasure set across pools.
 	totalDriveCount          int
@@ -127,7 +127,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		}
 	}
 
-	if cap(t.requestsPool) < apiRequestsMaxPerNode {
+	if cap(t.requestsPool) != apiRequestsMaxPerNode {
 		// Only replace if needed.
 		// Existing requests will use the previous limit,
 		// but new requests will use the new limit.
@@ -136,7 +136,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
 	}
 	t.requestsDeadline = cfg.RequestsDeadline
-	t.listQuorum = cfg.GetListQuorum()
+	t.listQuorum = cfg.ListQuorum
 	if globalReplicationPool != nil &&
 		cfg.ReplicationWorkers != t.replicationWorkers {
 		globalReplicationPool.ResizeFailedWorkers(cfg.ReplicationFailedWorkers)
@@ -170,7 +170,7 @@ func (t *apiConfig) shouldGzipObjects() bool {
 	return t.gzipObjects
 }
 
-func (t *apiConfig) getListQuorum() int {
+func (t *apiConfig) getListQuorum() string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
@@ -246,10 +246,12 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		globalHTTPStats.incS3RequestsIncoming()
 
-		if val := globalServiceFreeze.Load(); val != nil {
-			if unlock, ok := val.(chan struct{}); ok && unlock != nil {
-				// Wait until unfrozen.
-				<-unlock
+		if r.Header.Get(globalObjectPerfUserMetadata) == "" {
+			if val := globalServiceFreeze.Load(); val != nil {
+				if unlock, ok := val.(chan struct{}); ok && unlock != nil {
+					// Wait until unfrozen.
+					<-unlock
+				}
 			}
 		}
 

cmd/iam-object-store.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"path"
 	"strings"
 	"sync"
@@ -405,6 +406,136 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 	return nil
 }
 
+var (
+	usersListKey                   = "users/"
+	svcAccListKey                  = "service-accounts/"
+	groupsListKey                  = "groups/"
+	policiesListKey                = "policies/"
+	stsListKey                     = "sts/"
+	policyDBUsersListKey           = "policydb/users/"
+	policyDBSTSUsersListKey        = "policydb/sts-users/"
+	policyDBServiceAccountsListKey = "policydb/service-accounts/"
+	policyDBGroupsListKey          = "policydb/groups/"
+
+	allListKeys = []string{
+		usersListKey,
+		svcAccListKey,
+		groupsListKey,
+		policiesListKey,
+		stsListKey,
+		policyDBUsersListKey,
+		policyDBSTSUsersListKey,
+		policyDBServiceAccountsListKey,
+		policyDBGroupsListKey,
+	}
+)
+
+func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
+	res := make(map[string][]string)
+
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+
+		found := false
+		for _, listKey := range allListKeys {
+			if strings.HasPrefix(item.Item, listKey) {
+				found = true
+				name := strings.TrimPrefix(item.Item, listKey)
+				res[listKey] = append(res[listKey], name)
+				break
+			}
+		}
+
+		if !found && (item.Item != "format.json") {
+			logger.LogIf(ctx, fmt.Errorf("unknown type of IAM file listed: %v", item.Item))
+		}
+	}
+	return res, nil
+}
+
+// Assumes cache is locked by caller.
+func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
+	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Loads things in the same order as `LoadIAMCache()`
+
+	policiesList := listedConfigItems[policiesListKey]
+	for _, item := range policiesList {
+		policyName := path.Dir(item)
+		if err := iamOS.loadPolicyDoc(ctx, policyName, cache.iamPolicyDocsMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+
+	if iamOS.usersSysType == MinIOUsersSysType {
+
+		regUsersList := listedConfigItems[usersListKey]
+		for _, item := range regUsersList {
+			userName := path.Dir(item)
+			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+				return err
+			}
+		}
+
+		groupsList := listedConfigItems[groupsListKey]
+		for _, item := range groupsList {
+			group := path.Dir(item)
+			if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup {
+				return err
+			}
+		}
+	}
+
+	userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
+	for _, item := range userPolicyMappingsList {
+		userName := strings.TrimSuffix(item, ".json")
+		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+
+	groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
+	for _, item := range groupPolicyMappingsList {
+		groupName := strings.TrimSuffix(item, ".json")
+		if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+
+	svcAccList := listedConfigItems[svcAccListKey]
+	for _, item := range svcAccList {
+		userName := path.Dir(item)
+		if err := iamOS.loadUser(ctx, userName, svcUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+			return err
+		}
+	}
+
+	stsUsersList := listedConfigItems[stsListKey]
+	for _, item := range stsUsersList {
+		userName := path.Dir(item)
+		if err := iamOS.loadUser(ctx, userName, stsUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+			return err
+		}
+	}
+
+	stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey]
+	for _, item := range stsPolicyMappingsList {
+		stsName := strings.TrimSuffix(item, ".json")
+		if err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, cache.iamUserPolicyMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+
+	cache.buildUserGroupMemberships()
+	return nil
+}
+
 func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }

cmd/iam-store.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
@@ -127,27 +128,30 @@ func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string
 type UserIdentity struct {
 	Version     int              `json:"version"`
 	Credentials auth.Credentials `json:"credentials"`
+	UpdatedAt   time.Time        `json:"updatedAt,omitempty"`
 }
 
 func newUserIdentity(cred auth.Credentials) UserIdentity {
-	return UserIdentity{Version: 1, Credentials: cred}
+	return UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()}
 }
 
 // GroupInfo contains info about a group
 type GroupInfo struct {
-	Version int      `json:"version"`
-	Status  string   `json:"status"`
-	Members []string `json:"members"`
+	Version   int       `json:"version"`
+	Status    string    `json:"status"`
+	Members   []string  `json:"members"`
+	UpdatedAt time.Time `json:"updatedAt,omitempty"`
 }
 
 func newGroupInfo(members []string) GroupInfo {
-	return GroupInfo{Version: 1, Status: statusEnabled, Members: members}
+	return GroupInfo{Version: 1, Status: statusEnabled, Members: members, UpdatedAt: UTCNow()}
 }
 
 // MappedPolicy represents a policy name mapped to a user or group
 type MappedPolicy struct {
-	Version  int    `json:"version"`
-	Policies string `json:"policy"`
+	Version   int       `json:"version"`
+	Policies  string    `json:"policy"`
+	UpdatedAt time.Time `json:"updatedAt,omitempty"`
 }
 
 // converts a mapped policy into a slice of distinct policies
@@ -168,7 +172,7 @@ func (mp MappedPolicy) policySet() set.StringSet {
 }
 
 func newMappedPolicy(policy string) MappedPolicy {
-	return MappedPolicy{Version: 1, Policies: policy}
+	return MappedPolicy{Version: 1, Policies: policy, UpdatedAt: UTCNow()}
 }
 
 // PolicyDoc represents an IAM policy with some metadata.
@@ -314,26 +318,26 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
 // information in IAM (i.e sys.iam*Map) - this info is stored only in the STS
 // generated credentials. Thus we skip looking up group memberships, user map,
 // and group map and check the appropriate policy maps directly.
-func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]string, error) {
+func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]string, time.Time, error) {
 	if isGroup {
 		if mode == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[name]
 			if !ok {
-				return nil, errNoSuchGroup
+				return nil, time.Time{}, errNoSuchGroup
 			}
 
 			// Group is disabled, so we return no policy - this
 			// ensures the request is denied.
 			if g.Status == statusDisabled {
-				return nil, nil
+				return nil, time.Time{}, nil
 			}
 		}
 
-		return c.iamGroupPolicyMap[name].toSlice(), nil
+		return c.iamGroupPolicyMap[name].toSlice(), c.iamGroupPolicyMap[name].UpdatedAt, nil
 	}
 
 	if name == globalActiveCred.AccessKey {
-		return []string{"consoleAdmin"}, nil
+		return []string{"consoleAdmin"}, time.Time{}, nil
 	}
 
 	// When looking for a user's policies, we also check if the user
@@ -342,7 +346,7 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 	u, ok := c.iamUsersMap[name]
 	if ok {
 		if !u.IsValid() {
-			return nil, nil
+			return nil, time.Time{}, nil
 		}
 		parentName = u.ParentUser
 	}
@@ -353,7 +357,7 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 		if parentName == globalActiveCred.AccessKey && u.IsServiceAccount() {
 			// even if this is set, the claims present in the service
 			// accounts apply the final permissions if any.
-			return []string{"consoleAdmin"}, nil
+			return []string{"consoleAdmin"}, mp.UpdatedAt, nil
 		}
 		if parentName != "" {
 			mp = c.iamUserPolicyMap[parentName]
@@ -373,7 +377,7 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 		policies = append(policies, c.iamGroupPolicyMap[group].toSlice()...)
 	}
 
-	return policies, nil
+	return policies, mp.UpdatedAt, nil
 }
 
 // IAMStorageAPI defines an interface for the IAM persistence layer
@@ -432,48 +436,56 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 	cache := store.lock()
 	defer store.unlock()
 
-	if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
-		return err
-	}
-
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
-
-	if store.getUsersSysType() == MinIOUsersSysType {
-		if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
+	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
+		err := iamOS.loadAllFromObjStore(ctx, newCache)
+		if err != nil {
 			return err
 		}
-		if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
+	} else {
+
+		if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
 			return err
 		}
-	}
 
-	// load polices mapped to users
-	if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
-		return err
-	}
+		// Sets default canned policies, if none are set.
+		setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
 
-	// load policies mapped to groups
-	if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
-		return err
-	}
+		if store.getUsersSysType() == MinIOUsersSysType {
+			if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
+				return err
+			}
+			if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
+				return err
+			}
+		}
 
-	// load service accounts
-	if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
-		return err
-	}
+		// load polices mapped to users
+		if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
+			return err
+		}
 
-	// load STS temp users
-	if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
-		return err
-	}
+		// load policies mapped to groups
+		if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
+			return err
+		}
 
-	// load STS policy mappings
-	if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
-		return err
-	}
+		// load service accounts
+		if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
+			return err
+		}
 
-	newCache.buildUserGroupMemberships()
+		// load STS temp users
+		if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
+			return err
+		}
+
+		// load STS policy mappings
+		if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
+			return err
+		}
+
+		newCache.buildUserGroupMemberships()
+	}
 
 	cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap
 	cache.iamGroupsMap = newCache.iamGroupsMap
@@ -564,14 +576,14 @@ func (store *IAMStoreSys) PolicyDBGet(name string, isGroup bool, groups ...strin
 	cache := store.rlock()
 	defer store.runlock()
 
-	policies, err := cache.policyDBGet(store.getUsersSysType(), name, isGroup)
+	policies, _, err := cache.policyDBGet(store.getUsersSysType(), name, isGroup)
 	if err != nil {
 		return nil, err
 	}
 
 	if !isGroup {
 		for _, group := range groups {
-			ps, err := cache.policyDBGet(store.getUsersSysType(), group, true)
+			ps, _, err := cache.policyDBGet(store.getUsersSysType(), group, true)
 			if err != nil {
 				return nil, err
 			}
@@ -747,7 +759,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	cache := store.rlock()
 	defer store.runlock()
 
-	ps, err := cache.policyDBGet(store.getUsersSysType(), group, true)
+	ps, updatedAt, err := cache.policyDBGet(store.getUsersSysType(), group, true)
 	if err != nil {
 		return gd, err
 	}
@@ -756,8 +768,9 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 
 	if store.getUsersSysType() != MinIOUsersSysType {
 		return madmin.GroupDesc{
-			Name:   group,
-			Policy: policy,
+			Name:      group,
+			Policy:    policy,
+			UpdatedAt: updatedAt,
 		}, nil
 	}
 
@@ -767,10 +780,11 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	}
 
 	return madmin.GroupDesc{
-		Name:    group,
-		Status:  gi.Status,
-		Members: gi.Members,
-		Policy:  policy,
+		Name:      group,
+		Status:    gi.Status,
+		Members:   gi.Members,
+		Policy:    policy,
+		UpdatedAt: gi.UpdatedAt,
 	}, nil
 }
 
@@ -1074,6 +1088,33 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 	return ret, nil
 }
 
+// ListPolicyDocs - fetches all policy docs from storage and updates cache as well.
+// If bucketName is non-empty, returns policy docs matching the bucket.
+func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) {
+	cache := store.lock()
+	defer store.unlock()
+
+	m := map[string]PolicyDoc{}
+	err := store.loadPolicyDocs(ctx, m)
+	if err != nil {
+		return nil, err
+	}
+
+	// Sets default canned policies
+	setDefaultCannedPolicies(m)
+
+	cache.iamPolicyDocsMap = m
+
+	ret := map[string]PolicyDoc{}
+	for k, v := range m {
+		if bucketName == "" || v.Policy.MatchResource(bucketName) {
+			ret[k] = v
+		}
+	}
+
+	return ret, nil
+}
+
 // helper function - does not take locks.
 func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, iampolicy.Policy) {
 	var policies []string
@@ -1169,13 +1210,26 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
 				}
 				return madmin.AccountDisabled
 			}(),
-			MemberOf: cache.iamUserGroupMemberships[k].ToSlice(),
+			MemberOf:  cache.iamUserGroupMemberships[k].ToSlice(),
+			UpdatedAt: cache.iamUserPolicyMap[k].UpdatedAt,
 		}
 	}
 
 	return result
 }
 
+// GetUsersWithMappedPolicies - safely returns the name of access keys with associated policies
+func (store *IAMStoreSys) GetUsersWithMappedPolicies() map[string]string {
+	cache := store.rlock()
+	defer store.runlock()
+
+	result := make(map[string]string)
+	for k, v := range cache.iamUserPolicyMap {
+		result[k] = v.Policies
+	}
+	return result
+}
+
 // GetUserInfo - get info on a user.
 func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	if name == "" {
@@ -1202,6 +1256,7 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 		return madmin.UserInfo{
 			PolicyName: mappedPolicy.Policies,
 			MemberOf:   groups,
+			UpdatedAt:  mappedPolicy.UpdatedAt,
 		}, nil
 	}
 
@@ -1222,7 +1277,8 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 			}
 			return madmin.AccountDisabled
 		}(),
-		MemberOf: cache.iamUserGroupMemberships[name].ToSlice(),
+		MemberOf:  cache.iamUserGroupMemberships[name].ToSlice(),
+		UpdatedAt: cache.iamUserPolicyMap[name].UpdatedAt,
 	}, nil
 }
 
@@ -1448,30 +1504,71 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 	return nil
 }
 
-// GetAllParentUsers - returns all distinct "parent-users" associated with STS or service
-// credentials.
-func (store *IAMStoreSys) GetAllParentUsers() map[string]string {
+// ParentUserInfo contains extra info about a the parent user.
+type ParentUserInfo struct {
+	subClaimValue string
+	roleArns      set.StringSet
+}
+
+// GetAllParentUsers - returns all distinct "parent-users" associated with STS
+// or service credentials, mapped to all distinct roleARNs associated with the
+// parent user. The dummy role ARN is associated with parent users from
+// policy-claim based OpenID providers.
+func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	cache := store.rlock()
 	defer store.runlock()
 
-	res := map[string]string{}
+	res := map[string]ParentUserInfo{}
 	for _, cred := range cache.iamUsersMap {
-		if cred.IsServiceAccount() || cred.IsTemp() {
-			parentUser := cred.ParentUser
-			if cred.SessionToken != "" {
-				claims, err := getClaimsFromToken(cred.SessionToken)
-				if err != nil {
-					continue
-				}
-				if v, ok := claims[subClaim]; ok {
-					subFromToken, ok := v.(string)
-					if ok {
-						parentUser = subFromToken
-					}
-				}
+		// Only consider service account or STS credentials with
+		// non-empty session tokens.
+		if !(cred.IsServiceAccount() || cred.IsTemp()) ||
+			cred.SessionToken == "" {
+			continue
+		}
+
+		var (
+			err    error
+			claims map[string]interface{} = cred.Claims
+		)
+
+		if cred.IsServiceAccount() {
+			claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, cred.SecretKey)
+		} else if cred.IsTemp() {
+			claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
+		}
+
+		if err != nil {
+			continue
+		}
+		if cred.ParentUser == "" {
+			continue
+		}
+
+		subClaimValue := cred.ParentUser
+		if v, ok := claims[subClaim]; ok {
+			subFromToken, ok := v.(string)
+			if ok {
+				subClaimValue = subFromToken
 			}
-			if _, ok := res[parentUser]; !ok {
-				res[parentUser] = cred.ParentUser
+		}
+
+		roleArn := openid.DummyRoleARN.String()
+		s, ok := claims[roleArnClaim]
+		val, ok2 := s.(string)
+		if ok && ok2 {
+			roleArn = val
+		}
+		v, ok := res[cred.ParentUser]
+		if ok {
+			res[cred.ParentUser] = ParentUserInfo{
+				subClaimValue: subClaimValue,
+				roleArns:      v.roleArns.Union(set.CreateStringSet(roleArn)),
+			}
+		} else {
+			res[cred.ParentUser] = ParentUserInfo{
+				subClaimValue: subClaimValue,
+				roleArns:      set.CreateStringSet(roleArn),
 			}
 		}
 	}
@@ -1561,6 +1658,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errNoSuchServiceAccount
 	}
 
+	currentSecretKey := cr.SecretKey
 	if opts.secretKey != "" {
 		if !auth.IsSecretKeyValid(opts.secretKey) {
 			return auth.ErrInvalidSecretKeyLength
@@ -1582,20 +1680,21 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errors.New("unknown account status value")
 	}
 
-	if opts.sessionPolicy != nil {
-		m, err := getClaimsFromToken(cr.SessionToken)
-		if err != nil {
-			return fmt.Errorf("unable to get svc acc claims: %v", err)
-		}
+	m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey)
+	if err != nil {
+		return fmt.Errorf("unable to get svc acc claims: %v", err)
+	}
 
-		err = opts.sessionPolicy.Validate()
-		if err != nil {
+	if opts.sessionPolicy != nil {
+		if err := opts.sessionPolicy.Validate(); err != nil {
 			return err
 		}
+
 		policyBuf, err := json.Marshal(opts.sessionPolicy)
 		if err != nil {
 			return err
 		}
+
 		if len(policyBuf) > 16*humanize.KiByte {
 			return fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
@@ -1603,10 +1702,11 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		// Overwrite session policy claims.
 		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
 		m[iamPolicyClaimNameSA()] = "embedded-policy"
-		cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, globalActiveCred.SecretKey)
-		if err != nil {
-			return err
-		}
+	}
+
+	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
+	if err != nil {
+		return err
 	}
 
 	u := newUserIdentity(cr)
@@ -1619,6 +1719,24 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	return nil
 }
 
+// ListTempAccounts - lists only temporary accounts from the cache.
+func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+	cache := store.rlock()
+	defer store.runlock()
+
+	var tempAccounts []auth.Credentials
+	for _, v := range cache.iamUsersMap {
+		if v.IsTemp() && v.ParentUser == accessKey {
+			// Hide secret key & session key here
+			v.SecretKey = ""
+			v.SessionToken = ""
+			tempAccounts = append(tempAccounts, v)
+		}
+	}
+
+	return tempAccounts, nil
+}
+
 // ListServiceAccounts - lists only service accounts from the cache.
 func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
 	cache := store.rlock()

cmd/iam.go

@@ -29,6 +29,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
@@ -37,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -63,6 +65,13 @@ const (
 
 // IAMSys - config system.
 type IAMSys struct {
+	// Need to keep them here to keep alignment - ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	// metrics
+	LastRefreshTimeUnixNano         uint64
+	LastRefreshDurationMilliseconds uint64
+	TotalRefreshSuccesses           uint64
+	TotalRefreshFailures            uint64
+
 	sync.Mutex
 
 	iamRefreshInterval time.Duration
@@ -183,10 +192,17 @@ func (sys *IAMSys) Initialized() bool {
 
 // Load - loads all credentials, policies and policy mappings.
 func (sys *IAMSys) Load(ctx context.Context) error {
+	loadStartTime := time.Now()
 	err := sys.store.LoadIAMCache(ctx)
 	if err != nil {
+		atomic.AddUint64(&sys.TotalRefreshFailures, 1)
 		return err
 	}
+	loadDuration := time.Since(loadStartTime)
+
+	atomic.StoreUint64(&sys.LastRefreshDurationMilliseconds, uint64(loadDuration.Milliseconds()))
+	atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano()))
+	atomic.AddUint64(&sys.TotalRefreshSuccesses, 1)
 
 	select {
 	case <-sys.configLoaded:
@@ -198,6 +214,8 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
+	iamInitStart := time.Now()
+
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -268,6 +286,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	iamLoadStart := time.Now()
+
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx); err != nil {
@@ -318,20 +338,25 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	go sys.watch(ctx)
 
 	// Load RoleARN
-	if roleARN, rolePolicy, enabled := globalOpenIDConfig.GetRoleInfo(); enabled {
-		numPolicies := len(strings.Split(rolePolicy, ","))
-		validPolicies, _ := sys.store.FilterPolicies(rolePolicy, "")
-		numValidPolicies := len(strings.Split(validPolicies, ","))
-		if numPolicies != numValidPolicies {
-			logger.LogIf(ctx, fmt.Errorf("Some specified role policies (%s) were not defined - role based policies will not be enabled.", rolePolicy))
-			return
-		}
-		sys.rolesMap = map[arn.ARN]string{
-			roleARN: rolePolicy,
+	if rolePolicyMap := globalOpenIDConfig.GetRoleInfo(); rolePolicyMap != nil {
+		// Validate that policies associated with roles are defined.
+		for _, rolePolicies := range rolePolicyMap {
+			ps := newMappedPolicy(rolePolicies).toSlice()
+			numPolicies := len(ps)
+			validPolicies, _ := sys.store.FilterPolicies(rolePolicies, "")
+			numValidPolicies := len(strings.Split(validPolicies, ","))
+			if numPolicies != numValidPolicies {
+				logger.LogIf(ctx, fmt.Errorf("Some specified role policies (in %s) were not defined - role policies will not be enabled.", rolePolicies))
+				return
+			}
 		}
+		sys.rolesMap = rolePolicyMap
 	}
 
 	sys.printIAMRoles()
+
+	now := time.Now()
+	logger.Info("Finished loading IAM sub-system (took %.1fs of %.1fs to load data).", now.Sub(iamLoadStart).Seconds(), now.Sub(iamInitStart).Seconds())
 }
 
 // Prints IAM role ARNs.
@@ -363,22 +388,33 @@ func (sys *IAMSys) watch(ctx context.Context) {
 	if ok {
 		ch := watcher.watch(ctx, iamConfigPrefix)
 		for event := range ch {
-			// we simply log errors
-			err := sys.loadWatchedEvent(ctx, event)
-			logger.LogIf(ctx, err)
+			if err := sys.loadWatchedEvent(ctx, event); err != nil {
+				// we simply log errors
+				logger.LogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err))
+			}
 		}
 		return
 	}
 
+	var maxRefreshDurationSecondsForLog float64 = 10
+
 	// Fall back to loading all items periodically
 	ticker := time.NewTicker(sys.iamRefreshInterval)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-ticker.C:
+			refreshStart := time.Now()
 			if err := sys.Load(ctx); err != nil {
-				logger.LogIf(ctx, err)
+				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
+			} else {
+				took := time.Since(refreshStart).Seconds()
+				if took > maxRefreshDurationSecondsForLog {
+					// Log if we took a lot of time to load.
+					logger.Info("IAM refresh took %.2fs", took)
+				}
 			}
+
 		case <-ctx.Done():
 			return
 		}
@@ -436,16 +472,16 @@ func (sys *IAMSys) HasRolePolicy() bool {
 }
 
 // GetRolePolicy - returns policies associated with a role ARN.
-func (sys *IAMSys) GetRolePolicy(arnStr string) (string, error) {
-	arn, err := arn.Parse(arnStr)
+func (sys *IAMSys) GetRolePolicy(arnStr string) (arn.ARN, string, error) {
+	roleArn, err := arn.Parse(arnStr)
 	if err != nil {
-		return "", fmt.Errorf("RoleARN parse err: %v", err)
+		return arn.ARN{}, "", fmt.Errorf("RoleARN parse err: %v", err)
 	}
-	rolePolicy, ok := sys.rolesMap[arn]
+	rolePolicy, ok := sys.rolesMap[roleArn]
 	if !ok {
-		return "", fmt.Errorf("RoleARN %s is not defined.", arnStr)
+		return arn.ARN{}, "", fmt.Errorf("RoleARN %s is not defined.", arnStr)
 	}
-	return rolePolicy, nil
+	return roleArn, rolePolicy, nil
 }
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
@@ -504,9 +540,26 @@ func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[str
 		return nil, errServerNotInitialized
 	}
 
-	<-sys.configLoaded
+	select {
+	case <-sys.configLoaded:
+		return sys.store.ListPolicies(ctx, bucketName)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
 
-	return sys.store.ListPolicies(ctx, bucketName)
+// ListPolicyDocs - lists all canned policy docs.
+func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) {
+	if !sys.Initialized() {
+		return nil, errServerNotInitialized
+	}
+
+	select {
+	case <-sys.configLoaded:
+		return sys.store.ListPolicyDocs(ctx, bucketName)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
 }
 
 // SetPolicy - sets a new named policy.
@@ -639,25 +692,52 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 }
 
 // ListBucketUsers - list all users who can access this 'bucket'
-func (sys *IAMSys) ListBucketUsers(bucket string) (map[string]madmin.UserInfo, error) {
+func (sys *IAMSys) ListBucketUsers(ctx context.Context, bucket string) (map[string]madmin.UserInfo, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
-	<-sys.configLoaded
-
-	return sys.store.GetBucketUsers(bucket)
+	select {
+	case <-sys.configLoaded:
+		return sys.store.GetBucketUsers(bucket)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
 }
 
 // ListUsers - list all users.
-func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
+func (sys *IAMSys) ListUsers(ctx context.Context) (map[string]madmin.UserInfo, error) {
+	if !sys.Initialized() {
+		return nil, errServerNotInitialized
+	}
+	select {
+	case <-sys.configLoaded:
+		return sys.store.GetUsers(), nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// ListLDAPUsers - list LDAP users which has
+func (sys *IAMSys) ListLDAPUsers() (map[string]madmin.UserInfo, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
+	if sys.usersSysType != LDAPUsersSysType {
+		return nil, errIAMActionNotAllowed
+	}
+
 	<-sys.configLoaded
 
-	return sys.store.GetUsers(), nil
+	ldapUsers := make(map[string]madmin.UserInfo)
+	for user, policy := range sys.store.GetUsersWithMappedPolicies() {
+		ldapUsers[user] = madmin.UserInfo{
+			PolicyName: policy,
+			Status:     madmin.AccountEnabled,
+		}
+	}
+	return ldapUsers, nil
 }
 
 // IsTempUser - returns if given key is a temporary user.
@@ -799,14 +879,17 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		}
 	}
 
-	var cred auth.Credentials
-
+	var accessKey, secretKey string
 	var err error
 	if len(opts.accessKey) > 0 {
-		cred, err = auth.CreateNewCredentialsWithMetadata(opts.accessKey, opts.secretKey, m, globalActiveCred.SecretKey)
+		accessKey, secretKey = opts.accessKey, opts.secretKey
 	} else {
-		cred, err = auth.GetNewCredentialsWithMetadata(m, globalActiveCred.SecretKey)
+		accessKey, secretKey, err = auth.GenerateCredentials()
+		if err != nil {
+			return auth.Credentials{}, err
+		}
 	}
+	cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey)
 	if err != nil {
 		return auth.Credentials{}, err
 	}
@@ -850,9 +933,26 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 		return nil, errServerNotInitialized
 	}
 
-	<-sys.configLoaded
+	select {
+	case <-sys.configLoaded:
+		return sys.store.ListServiceAccounts(ctx, accessKey)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
 
-	return sys.store.ListServiceAccounts(ctx, accessKey)
+// ListTempAccounts - lists all services accounts associated to a specific user
+func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+	if !sys.Initialized() {
+		return nil, errServerNotInitialized
+	}
+
+	select {
+	case <-sys.configLoaded:
+		return sys.store.ListTempAccounts(ctx, accessKey)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
 }
 
 // GetServiceAccount - wrapper method to get information about a service account
@@ -880,9 +980,12 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 
 	var embeddedPolicy *iampolicy.Policy
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return auth.Credentials{}, nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
 	}
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
 	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
@@ -915,9 +1018,12 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errNoSuchServiceAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return jwtClaims.Map(), nil
 }
@@ -1001,10 +1107,27 @@ func (sys *IAMSys) SetUserSecretKey(ctx context.Context, accessKey string, secre
 // purgeExpiredCredentialsForExternalSSO - validates if local credentials are still valid
 // by checking remote IDP if the relevant users are still active and present.
 func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
-	parentUsers := sys.store.GetAllParentUsers()
+	parentUsersMap := sys.store.GetAllParentUsers()
 	var expiredUsers []string
-	for parentUser, expiredUser := range parentUsers {
-		u, err := globalOpenIDConfig.LookupUser(parentUser)
+	for parentUser, puInfo := range parentUsersMap {
+		// There are multiple role ARNs for parent user only when there
+		// are multiple openid provider configurations with the same ID
+		// provider. We lookup the provider associated with some one of
+		// the roleARNs to check if the user still exists. If they don't
+		// we can safely remove credentials for this parent user
+		// associated with any of the provider configurations.
+		//
+		// If there is no roleARN mapped to the user, the user may be
+		// coming from a policy claim based openid provider.
+		roleArns := puInfo.roleArns.ToSlice()
+		var roleArn string
+		if len(roleArns) == 0 {
+			logger.LogIf(GlobalContext,
+				fmt.Errorf("parentUser: %s had no roleArns mapped!", parentUser))
+			continue
+		}
+		roleArn = roleArns[0]
+		u, err := globalOpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue
@@ -1012,7 +1135,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 		// If user is set to "disabled", we will remove them
 		// subsequently.
 		if !u.Enabled {
-			expiredUsers = append(expiredUsers, expiredUser)
+			expiredUsers = append(expiredUsers, parentUser)
 		}
 	}
 
@@ -1025,12 +1148,12 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var allDistNames []string
-	for parentUser, expiredUser := range parentUsers {
+	for parentUser := range parentUsers {
 		if !globalLDAPConfig.IsLDAPUserDN(parentUser) {
 			continue
 		}
 
-		allDistNames = append(allDistNames, expiredUser)
+		allDistNames = append(allDistNames, parentUser)
 	}
 
 	expiredUsers, err := globalLDAPConfig.GetNonEligibleUserDistNames(allDistNames)
@@ -1063,12 +1186,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 		if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok {
 			// Try to find the ldapUsername for this
 			// parentUser by extracting JWT claims
-			jwtClaims, err := auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
-			if err != nil {
-				// skip this cred - session token seems
-				// invalid
+			var (
+				jwtClaims *jwt.MapClaims
+				err       error
+			)
+
+			if cred.SessionToken == "" {
 				continue
 			}
+
+			if cred.IsServiceAccount() {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, cred.SecretKey)
+				if err != nil {
+					jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+				}
+			} else {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+			if err != nil {
+				// skip this cred - session token seems invalid
+				continue
+			}
+
 			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
 			if !ok {
 				// skip this cred - we dont have the
@@ -1230,9 +1369,12 @@ func (sys *IAMSys) ListGroups(ctx context.Context) (r []string, err error) {
 		return r, errServerNotInitialized
 	}
 
-	<-sys.configLoaded
-
-	return sys.store.ListGroups(ctx)
+	select {
+	case <-sys.configLoaded:
+		return sys.store.ListGroups(ctx)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
 }
 
 // PolicyDBSet - sets a policy for a user or group in the PolicyDB.

cmd/jwt.go

@@ -132,7 +132,7 @@ func authenticateURL(accessKey, secretKey string) (string, error) {
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
+func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
 	token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {

cmd/jwt_test.go

@@ -149,7 +149,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, _, _, gotErr := webRequestAuthenticate(testCase.req)
+		_, _, _, gotErr := metricsRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}

cmd/metacache-bucket_test.go

@@ -42,7 +42,7 @@ func Benchmark_bucketMetacache_findCache(b *testing.B) {
 			FilterPrefix: "",
 			Marker:       "",
 			Limit:        0,
-			AskDisks:     0,
+			AskDisks:     "strict",
 			Recursive:    false,
 			Separator:    slashSeparator,
 			Create:       true,
@@ -59,7 +59,7 @@ func Benchmark_bucketMetacache_findCache(b *testing.B) {
 			FilterPrefix: "",
 			Marker:       "",
 			Limit:        0,
-			AskDisks:     0,
+			AskDisks:     "strict",
 			Recursive:    false,
 			Separator:    slashSeparator,
 			Create:       true,

cmd/metacache-entries.go

@@ -281,10 +281,16 @@ func (m metaCacheEntries) shallowClone() metaCacheEntries {
 }
 
 type metadataResolutionParams struct {
-	dirQuorum int    // Number if disks needed for a directory to 'exist'.
-	objQuorum int    // Number of disks needed for an object to 'exist'.
-	bucket    string // Name of the bucket. Used for generating cached fileinfo.
-	strict    bool   // Versions must match exactly, including all metadata.
+	dirQuorum int // Number if disks needed for a directory to 'exist'.
+	objQuorum int // Number of disks needed for an object to 'exist'.
+
+	// An optimization request only an 'n' amount of versions from xl.meta
+	// to avoid resolving all versions to figure out the latest 'version'
+	// for ListObjects, ListObjectsV2
+	requestedVersions int
+
+	bucket string // Name of the bucket. Used for generating cached fileinfo.
+	strict bool   // Versions must match exactly, including all metadata.
 
 	// Reusable slice for resolution
 	candidates [][]xlMetaV2ShallowVersion
@@ -372,7 +378,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 		reusable: true,
 		cached:   &xlMetaV2{metaV: selected.cached.metaV},
 	}
-	selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.candidates...)
+	selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.requestedVersions, r.candidates...)
 	if len(selected.cached.versions) == 0 {
 		return nil, false
 	}
@@ -498,7 +504,7 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 	return versions
 }
 
-// fileInfoVersions converts the metadata to FileInfoVersions where possible.
+// fileInfos converts the metadata to ObjectInfo where possible.
 // Metadata that cannot be decoded is skipped.
 func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (objects []ObjectInfo) {
 	objects = make([]ObjectInfo, 0, m.len())

cmd/metacache-server-pool.go

@@ -28,6 +28,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -110,7 +112,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	// If we don't have a list id we must ask the server if it has a cache or create a new.
 	if o.ID != "" && !o.Transient {
 		// Create or ping with handout...
-		rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 		defer cancel()
 		var c *metacache
@@ -198,7 +200,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 		}
 		entries.truncate(0)
 		go func() {
-			rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+			rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 			if rpc != nil {
 				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
 				defer cancel()
@@ -289,6 +291,14 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	}
 	mu.Unlock()
 
+	// Do lifecycle filtering.
+	if o.Lifecycle != nil {
+		filterIn := make(chan metaCacheEntry, 10)
+		go filterLifeCycle(ctx, o.Bucket, *o.Lifecycle, o.Retention, filterIn, results)
+		// Replace results.
+		results = filterIn
+	}
+
 	// Gather results to a single channel.
 	err := mergeEntryChannels(ctx, inputs, results, func(existing, other *metaCacheEntry) (replace bool) {
 		// Pick object over directory
@@ -298,21 +308,20 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 		if !existing.isDir() && other.isDir() {
 			return false
 		}
-
-		eFIV, err := existing.fileInfo(o.Bucket)
+		eMeta, err := existing.xlmeta()
 		if err != nil {
 			return true
 		}
-		oFIV, err := other.fileInfo(o.Bucket)
+		oMeta, err := other.xlmeta()
 		if err != nil {
 			return false
 		}
 		// Replace if modtime is newer
-		if !oFIV.ModTime.Equal(eFIV.ModTime) {
-			return oFIV.ModTime.After(eFIV.ModTime)
+		if !oMeta.latestModtime().Equal(oMeta.latestModtime()) {
+			return oMeta.latestModtime().After(eMeta.latestModtime())
 		}
 		// Use NumVersions as a final tiebreaker.
-		return oFIV.NumVersions > eFIV.NumVersions
+		return len(oMeta.versions) > len(eMeta.versions)
 	})
 
 	cancelList()
@@ -346,6 +355,46 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	return nil
 }
 
+// filterLifeCycle will filter out objects if the most recent
+// version should be deleted by lifecycle.
+// out will be closed when there are no more results.
+// When 'in' is closed or the context is canceled the
+// function closes 'out' and exits.
+func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle, lr lock.Retention, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
+	defer close(out)
+	for {
+		var obj metaCacheEntry
+		var ok bool
+		select {
+		case <-ctx.Done():
+			return
+		case obj, ok = <-in:
+			if !ok {
+				return
+			}
+		}
+
+		fi, err := obj.fileInfo(bucket)
+		if err != nil {
+			continue
+		}
+		objInfo := fi.ToObjectInfo(bucket, obj.name)
+		action := evalActionFromLifecycle(ctx, lc, lr, objInfo, false)
+		switch action {
+		case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+			fallthrough
+		case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+			// Skip this entry.
+			continue
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case out <- obj:
+		}
+	}
+}
+
 func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {
 	// Use ID as the object name...
 	o.pool = z.getAvailablePoolIdx(ctx, minioMetaBucket, o.ID, 10<<20)
@@ -369,7 +418,7 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 	filteredResults := o.gatherResults(ctx, outCh)
 
 	mc := o.newMetacache()
-	meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(o.Bucket), o: *o}
+	meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)), o: *o}
 
 	// Save listing...
 	go func() {

cmd/metacache-set.go

@@ -32,6 +32,8 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
@@ -64,10 +66,8 @@ type listPathOptions struct {
 	// Limit the number of results.
 	Limit int
 
-	// The number of disks to ask. Special values:
-	// 0 uses default number of disks.
-	// -1 use at least 50% of disks or at least the default number.
-	AskDisks int
+	// The number of disks to ask.
+	AskDisks string
 
 	// InclDeleted will keep all entries where latest version is a delete marker.
 	InclDeleted bool
@@ -94,6 +94,14 @@ type listPathOptions struct {
 	// Versioned is this a ListObjectVersions call.
 	Versioned bool
 
+	// Lifecycle performs filtering based on lifecycle.
+	// This will filter out objects if the most recent version should be deleted by lifecycle.
+	// Is not transferred across request calls.
+	Lifecycle *lifecycle.Lifecycle
+
+	// Retention configuration, needed to be passed along with lifecycle if set.
+	Retention lock.Retention
+
 	// pool and set of where the cache is located.
 	pool, set int
 }
@@ -189,7 +197,11 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 		}
 		if resCh != nil {
 			resErr = io.EOF
-			resCh <- results
+			select {
+			case <-ctx.Done():
+				// Nobody wants it.
+			case resCh <- results:
+			}
 		}
 	}()
 	return func() (metaCacheEntriesSorted, error) {
@@ -351,7 +363,7 @@ func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSor
 
 func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
 	retries := 0
-	rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+	rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 
 	for {
 		if contextCanceled(ctx) {
@@ -531,21 +543,43 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 	}
 }
 
+// getListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func getListQuorum(quorum string, driveCount int) int {
+	switch quorum {
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "reduced":
+		return 2
+	case "strict":
+		return driveCount
+	}
+	// Defaults to (driveCount+1)/2 drives per set, defaults to "optimal" value
+	if driveCount > 0 {
+		return (driveCount + 1) / 2
+	} // "3" otherwise.
+	return 3
+}
+
 // Will return io.EOF if continuing would not yield more results.
 func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
 	defer close(results)
 	o.debugf(color.Green("listPath:")+" with options: %#v", o)
 
-	askDisks := o.AskDisks
-	listingQuorum := o.AskDisks - 1
-	disks := er.getDisks()
+	// get non-healing disks for listing
+	disks, _ := er.getOnlineDisksWithHealing()
+	askDisks := getListQuorum(o.AskDisks, er.setDriveCount)
 	var fallbackDisks []StorageAPI
 
 	// Special case: ask all disks if the drive count is 4
-	if askDisks <= 0 || er.setDriveCount == 4 {
-		askDisks = len(disks)          // with 'strict' quorum list on all online disks.
-		listingQuorum = len(disks) / 2 // keep this such that we can list all objects with different quorum ratio.
+	if er.setDriveCount == 4 || askDisks > len(disks) {
+		askDisks = len(disks) // use all available drives
 	}
+
+	// However many we ask, versions must exist on ~50%
+	listingQuorum := (askDisks + 1) / 2
+
 	if askDisks > 0 && len(disks) > askDisks {
 		rand.Shuffle(len(disks), func(i, j int) {
 			disks[i], disks[j] = disks[j], disks[i]
@@ -561,6 +595,13 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 		bucket:    o.Bucket,
 	}
 
+	// Maximum versions requested for "latest" object
+	// resolution on versioned buckets, this is to be only
+	// used when o.Versioned is false
+	if !o.Versioned {
+		resolver.requestedVersions = 1
+	}
+
 	ctxDone := ctx.Done()
 	return listPathRaw(ctx, listPathRawOptions{
 		disks:         disks,
@@ -789,14 +830,30 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	fallback := func(err error) bool {
+	// Keep track of fallback disks
+	var fdMu sync.Mutex
+	fds := opts.fallbackDisks
+	fallback := func(err error) StorageAPI {
 		switch err.(type) {
 		case StorageErr:
-			// all supported disk errors
-			// attempt a fallback.
-			return true
+			// Attempt to grab a fallback disk
+			fdMu.Lock()
+			defer fdMu.Unlock()
+			if len(fds) == 0 {
+				return nil
+			}
+			fdsCopy := fds
+			for _, fd := range fdsCopy {
+				// Grab a fallback disk
+				fds = fds[1:]
+				if fd != nil && fd.IsOnline() {
+					return fd
+				}
+			}
 		}
-		return false
+		// Either no more disks for fallback or
+		// not a storage error.
+		return nil
 	}
 	askDisks := len(disks)
 	readers := make([]*metacacheReader, askDisks)
@@ -825,25 +882,20 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			}
 
 			// fallback only when set.
-			if len(opts.fallbackDisks) > 0 && fallback(werr) {
+			for fd := fallback(werr); fd != nil; {
 				// This fallback is only set when
 				// askDisks is less than total
 				// number of disks per set.
-				for _, fd := range opts.fallbackDisks {
-					if fd == nil {
-						continue
-					}
-					werr = fd.WalkDir(ctx, WalkDirOptions{
-						Bucket:         opts.bucket,
-						BaseDir:        opts.path,
-						Recursive:      opts.recursive,
-						ReportNotFound: opts.reportNotFound,
-						FilterPrefix:   opts.filterPrefix,
-						ForwardTo:      opts.forwardTo,
-					}, w)
-					if werr == nil {
-						break
-					}
+				werr = fd.WalkDir(ctx, WalkDirOptions{
+					Bucket:         opts.bucket,
+					BaseDir:        opts.path,
+					Recursive:      opts.recursive,
+					ReportNotFound: opts.reportNotFound,
+					FilterPrefix:   opts.filterPrefix,
+					ForwardTo:      opts.forwardTo,
+				}, w)
+				if werr == nil {
+					break
 				}
 			}
 			w.CloseWithError(werr)

cmd/metrics-v2.go

@@ -62,6 +62,7 @@ func init() {
 		getS3TTFBMetric(),
 		getILMNodeMetrics(),
 		getScannerNodeMetrics(),
+		getIAMNodeMetrics(),
 	}
 
 	allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -121,6 +122,7 @@ const (
 	quotaSubsystem            MetricSubsystem = "quota"
 	ilmSubsystem              MetricSubsystem = "ilm"
 	scannerSubsystem          MetricSubsystem = "scanner"
+	iamSubsystem              MetricSubsystem = "iam"
 )
 
 // MetricName are the individual names for the metric.
@@ -1266,6 +1268,62 @@ func getScannerNodeMetrics() *MetricsGroup {
 	return mg
 }
 
+func getIAMNodeMetrics() *MetricsGroup {
+	mg := &MetricsGroup{}
+	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
+		lastSyncTime := atomic.LoadUint64(&globalIAMSys.LastRefreshTimeUnixNano)
+		var sinceLastSyncMillis uint64
+		if lastSyncTime != 0 {
+			sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond)
+		}
+
+		metrics = []Metric{
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "last_sync_duration_millis",
+					Help:      "Last successful IAM data sync duration in milliseconds",
+					Type:      gaugeMetric,
+				},
+				Value: float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "since_last_sync_millis",
+					Help:      "Time (in milliseconds) since last successful IAM data sync. This is set to 0 until the first sync after server start.",
+					Type:      gaugeMetric,
+				},
+				Value: float64(sinceLastSyncMillis),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "sync_successes",
+					Help:      "Number of successful IAM data syncs since server start.",
+					Type:      counterMetric,
+				},
+				Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshSuccesses)),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "sync_failures",
+					Help:      "Number of failed IAM data syncs since server start.",
+					Type:      counterMetric,
+				},
+				Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)),
+			},
+		}
+		return metrics
+	})
+	return mg
+}
+
 func getMinioVersionMetrics() *MetricsGroup {
 	mg := &MetricsGroup{}
 	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
@@ -1651,13 +1709,13 @@ func getClusterTierMetrics() *MetricsGroup {
 		cacheInterval: 10 * time.Second,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		if globalTierConfigMgr.Empty() {
-			return
-		}
 		objLayer := newObjectLayerFn()
 		if objLayer == nil || globalIsGateway {
 			return
 		}
+		if globalTierConfigMgr.Empty() {
+			return
+		}
 
 		dui, err := loadDataUsageFromBackend(GlobalContext, objLayer)
 		if err != nil {

cmd/metrics.go

@@ -191,7 +191,7 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
 // collects gateway specific metrics for MinIO instance in Prometheus specific format
 // and sends to given channel
 func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
-	if !globalIsGateway || (globalGatewayName != S3BackendGateway && globalGatewayName != AzureBackendGateway && globalGatewayName != GCSBackendGateway) {
+	if !globalIsGateway || (globalGatewayName != S3BackendGateway) {
 		return
 	}
 
@@ -674,7 +674,7 @@ func metricsHandler() http.Handler {
 // AuthMiddleware checks if the bearer token is valid and authorized.
 func AuthMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		claims, groups, owner, authErr := webRequestAuthenticate(r)
+		claims, groups, owner, authErr := metricsRequestAuthenticate(r)
 		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
 			w.WriteHeader(http.StatusForbidden)
 			return

cmd/mrf.go

@@ -185,7 +185,7 @@ func (m *mrfState) healRoutine() {
 	defer idler.Stop()
 
 	mrfHealingOpts := madmin.HealOpts{
-		ScanMode: globalHealConfig.ScanMode(),
+		ScanMode: madmin.HealNormalScan,
 		Remove:   healDeleteDangling,
 	}
 

cmd/namespace-lock.go

@@ -170,6 +170,10 @@ func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout
 	}) {
 		timeout.LogFailure()
 		cancel()
+		switch err := newCtx.Err(); err {
+		case context.Canceled:
+			return LockContext{ctx: ctx, cancel: func() {}}, err
+		}
 		return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
@@ -195,6 +199,10 @@ func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeou
 	}) {
 		timeout.LogFailure()
 		cancel()
+		switch err := newCtx.Err(); err {
+		case context.Canceled:
+			return LockContext{ctx: ctx, cancel: func() {}}, err
+		}
 		return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
@@ -247,6 +255,10 @@ func (li *localLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeou
 					li.ns.unlock(li.volume, li.paths[si], readLock)
 				}
 			}
+			switch err := ctx.Err(); err {
+			case context.Canceled:
+				return LockContext{}, err
+			}
 			return LockContext{}, OperationTimedOut{}
 		}
 		success[i] = 1
@@ -280,6 +292,10 @@ func (li *localLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeo
 					li.ns.unlock(li.volume, li.paths[si], readLock)
 				}
 			}
+			switch err := ctx.Err(); err {
+			case context.Canceled:
+				return LockContext{}, err
+			}
 			return LockContext{}, OperationTimedOut{}
 		}
 		success[i] = 1