fix: update storage-class properly

fixes #14005

.github/workflows/go-healing.yml

@@ -0,0 +1,48 @@
+name: Healing Functional Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
+          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
+          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
+          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_AUTO_ENCRYPTION: on
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make verify-healing

.github/workflows/go.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   build:
-    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -46,4 +46,3 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify
-          make verify-healing

.gitignore

@@ -25,6 +25,11 @@ node_modules/
 mc.*
 s3-check-md5*
 xl-meta*
-healing-bin*
+healing-*
 inspect*
 200M*
+hash-set
+minio.RELEASE*
+mc
+nancy
+inspects/*

COMPLIANCE.md

@@ -0,0 +1,6 @@
+# AGPLv3 Compliance
+We have designed MinIO as an Open Source software for the Open Source software community. This requires applications to consider whether their usage of MinIO is in compliance with the GNU AGPLv3 [license](https://github.com/minio/minio/blob/master/LICENSE).
+
+MinIO cannot make the determination as to whether your application's usage of MinIO is in compliance with the AGPLv3 license requirements. You should instead rely on your own legal counsel or licensing specialists to audit and ensure your application is in compliance with the licenses of MinIO and all other open-source projects with which your application integrates or interacts. We understand that AGPLv3 licensing is complex and nuanced. It is for that reason we strongly encourage using experts in licensing to make any such determinations around compliance instead of relying on apocryphal or anecdotal advice.
+
+[MinIO Commercial Licensing](https://min.io/pricing) is the best option for applications that trigger AGPLv3 obligations (e.g. open sourcing your application). Applications using MinIO - or any other OSS-licensed code - without validating their usage do so at their own risk.

Dockerfile

@@ -1,6 +1,7 @@
 FROM minio/minio:latest
 
 COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Dockerfile.dev

@@ -1,6 +1,7 @@
 FROM minio/minio:latest
 
 COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Makefile

@@ -73,6 +73,7 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
+	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"

README.md

@@ -237,13 +237,11 @@ mc admin update <minio alias, e.g., myminio>
 
 ## Important things to remember during MinIO upgrades
 
+- Container environments are advised to update the container images instead of updating binaries inside the container.
+- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
-- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
-- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
 - If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
-- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
-- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
 # Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
@@ -257,5 +255,6 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 # License
-MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
-MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
+- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)

buildscripts/unaligned-healing.sh

@@ -0,0 +1,171 @@
+#!/bin/bash -e
+#
+
+set -E
+set -o pipefail
+set -x
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
+	chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
+    fi
+}
+
+function start_minio_16drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
+    "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
+    "${WORK_DIR}/mc" cp \
+		     "${WORK_DIR}/unaligned" \
+		     minio/healing-shard-bucket/unaligned \
+		     --disable-multipart --quiet
+
+    ## "unaligned" object name gets consistently distributed
+    ## to disks in following distribution order
+    ##
+    ## NOTE: if you change the name make sure to change the
+    ## distribution order present here
+    ##
+    ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+
+    ## make sure to remove the "last" data shard
+    rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
+    sleep 10
+    ## Heal the shard
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+    ## then remove any other data shard let's pick first disk
+    ## - 1st data shard.
+    rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
+    sleep 10
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_16drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/verify-healing.sh

@@ -21,51 +21,70 @@ function start_minio_3_node() {
     start_port=$2
     args=""
     for i in $(seq 1 3); do
-        args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
+	args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
     "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    disown $!
+    pid1=$!
+    disown ${pid1}
 
     "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    disown $!
+    pid2=$!
+    disown $pid2
 
     "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    disown $!
+    pid3=$!
+    disown $pid3
 
     sleep "$1"
-    if [ "$(pgrep -c minio)" -ne 3 ]; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+
+    if ! ps -p $pid1 1>&2 > /dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/dist-minio-server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
+
+    if ! ps -p $pid2 1>&2 > /dev/null; then
+	echo "server2 log:"
+	cat "${WORK_DIR}/dist-minio-server2.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ps -p $pid3 1>&2 > /dev/null; then
+	echo "server3 log:"
+	cat "${WORK_DIR}/dist-minio-server3.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
     if ! pkill minio; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+	for i in $(seq 1 3); do
+	    echo "server$i log:"
+	    cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
 
     sleep 1;
     if pgrep minio; then
-        # forcibly killing, to proceed further properly.
-        if ! pkill -9 minio; then
-            echo "no minio process running anymore, proceed."
-        fi
+	# forcibly killing, to proceed further properly.
+	if ! pkill -9 minio; then
+	    echo "no minio process running anymore, proceed."
+	fi
     fi
 }
 
 
 function check_online() {
     if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
-        echo "1"
+	echo "1"
     fi
 }
 
@@ -96,14 +115,14 @@ function perform_test() {
 
     rv=$(check_online)
     if [ "$rv" == "1" ]; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        pkill -9 minio
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+	for i in $(seq 1 3); do
+	    echo "server$i log:"
+	    cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	pkill -9 minio
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
 }
 

cmd/admin-handler-utils.go

@@ -86,6 +86,8 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			Description:    e.Message,
 			HTTPStatusCode: e.StatusCode,
 		}
+	case SRError:
+		apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause)
 	default:
 		switch {
 		case errors.Is(err, errConfigNotFound):

cmd/admin-handlers-site-replication.go

@@ -45,16 +45,16 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	}
 
 	var sites []madmin.PeerSite
-	errCode := readJSONBody(ctx, r.Body, &sites, cred.SecretKey)
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	status, errInfo := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
-	if errInfo.Code != ErrNone {
-		logger.LogIf(ctx, errInfo)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
@@ -67,12 +67,12 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, body)
 }
 
-// SRInternalJoin - PUT /minio/admin/v3/site-replication/join
+// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
 //
 // used internally to tell current cluster to enable SR with
 // the provided peer clusters and service account.
-func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalJoin")
+func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerJoin")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -81,24 +81,22 @@ func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	var joinArg madmin.SRInternalJoinReq
-	errCode := readJSONBody(ctx, r.Body, &joinArg, cred.SecretKey)
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	var joinArg madmin.SRPeerJoinReq
+	if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	errInfo := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg)
-	if errInfo.Code != ErrNone {
-		logger.LogIf(ctx, errInfo)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+	if err := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
 
-// SRInternalBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
-func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalBucketOps")
+// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
+func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerBucketOps")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -114,7 +112,7 @@ func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Req
 	var err error
 	switch operation {
 	default:
-		err = errInvalidArgument
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.MakeWithVersioningBktOp:
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
@@ -139,9 +137,9 @@ func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Req
 
 }
 
-// SRInternalReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
-func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
+func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateIAMItem")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -151,16 +149,15 @@ func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *h
 	}
 
 	var item madmin.SRIAMItem
-	errCode := readJSONBody(ctx, r.Body, &item, "")
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	var err error
 	switch item.Type {
 	default:
-		err = errInvalidArgument
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRIAMItemPolicy:
 		if item.Policy == nil {
 			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
@@ -190,9 +187,9 @@ func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *h
 	}
 }
 
-// SRInternalReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
-func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalReplicateBucketItem")
+// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateBucketItem")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -202,16 +199,15 @@ func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r
 	}
 
 	var item madmin.SRBucketMeta
-	errCode := readJSONBody(ctx, r.Body, &item, "")
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	var err error
 	switch item.Type {
 	default:
-		err = errInvalidArgument
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRBucketMetaTypePolicy:
 		if item.Policy == nil {
 			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
@@ -276,7 +272,7 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
 	}
 }
 
-func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -293,24 +289,25 @@ func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *htt
 	}
 }
 
-func readJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) APIErrorCode {
+func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
 	data, err := ioutil.ReadAll(body)
 	if err != nil {
-		return ErrInvalidRequest
+		return SRError{
+			Cause: err,
+			Code:  ErrSiteReplicationInvalidRequest,
+		}
 	}
 
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return ErrInvalidRequest
+			return SRError{
+				Cause: err,
+				Code:  ErrSiteReplicationInvalidRequest,
+			}
 		}
 	}
 
-	err = json.Unmarshal(data, v)
-	if err != nil {
-		return ErrAdminConfigBadJSON
-	}
-
-	return ErrNone
+	return json.Unmarshal(data, v)
 }

cmd/admin-handlers-users.go

@@ -58,18 +58,10 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalIAMSys.DeleteUser(ctx, accessKey); err != nil {
+	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to delete user.
-	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // ListUsers - GET /minio/admin/v3/list-users?bucket={bucket}
@@ -424,14 +416,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var uinfo madmin.UserInfo
-	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+	var ureq madmin.AddOrUpdateUserReq
+	if err = json.Unmarshal(configBytes, &ureq); err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
-	if err = globalIAMSys.CreateUser(ctx, accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -978,17 +970,11 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount)
+	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	for _, nerr := range globalNotificationSys.DeleteServiceAccount(serviceAccount) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 
 	// Call site replication hook. Only LDAP accounts are supported for
 	// replication operations.
@@ -1107,19 +1093,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	accountName := cred.AccessKey
-	var policies []string
-	switch globalIAMSys.usersSysType {
-	case MinIOUsersSysType:
-		policies, err = globalIAMSys.PolicyDBGet(accountName, false)
-	case LDAPUsersSysType:
-		parentUser := accountName
-		if cred.ParentUser != "" {
-			parentUser = cred.ParentUser
-		}
-		policies, err = globalIAMSys.PolicyDBGet(parentUser, false, cred.Groups...)
-	default:
-		err = errors.New("should never happen")
+	if cred.IsTemp() || cred.IsServiceAccount() {
+		// For derived credentials, check the parent user's permissions.
+		accountName = cred.ParentUser
 	}
+	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1198,6 +1176,16 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 }
 
 // InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
+//
+// Newer API response with policy timestamps is returned with query parameter
+// `v=2` like:
+//
+// GET /minio/admin/v3/info-canned-policy?name={policyName}&v=2
+//
+// The newer API will eventually become the default (and only) one. The older
+// response is to return only the policy JSON. The newer response returns
+// timestamps along with the policy JSON. Both versions are supported for now,
+// for smooth transition to new API.
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
@@ -1208,13 +1196,36 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	name := mux.Vars(r)["name"]
+	policies := newMappedPolicy(name).toSlice()
+	if len(policies) != 1 {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
+		return
+	}
+
+	policyDoc, err := globalIAMSys.InfoPolicy(name)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	buf, err := json.MarshalIndent(policy, "", " ")
+	// Is the new API version being requested?
+	infoPolicyAPIVersion := r.Form.Get("v")
+	if infoPolicyAPIVersion == "2" {
+		buf, err := json.MarshalIndent(policyDoc, "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		w.Write(buf)
+		return
+	} else if infoPolicyAPIVersion != "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("invalid version parameter 'v' supplied")), r.URL)
+		return
+	}
+
+	// Return the older API response value of just the policy json.
+	buf, err := json.MarshalIndent(policyDoc.Policy, "", " ")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers-users_test.go

@@ -18,8 +18,15 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
@@ -29,7 +36,9 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
 )
 
@@ -177,6 +186,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
 
 				suite.SetUpSuite(c)
 				suite.TestUserCreate(c)
+				suite.TestUserPolicyEscalationBug(c)
 				suite.TestPolicyCreate(c)
 				suite.TestCannedPolicies(c)
 				suite.TestGroupAddRemove(c)
@@ -222,6 +232,24 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 		c.Fatalf("user could not create bucket: %v", err)
 	}
 
+	// 3.10. Check that user's password can be updated.
+	_, newSecretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to update user's secret key: %v", err)
+	}
+	// 3.10.1 Check that old password no longer works.
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err == nil {
+		c.Fatalf("user was unexpectedly able to create bucket with bad password!")
+	}
+	// 3.10.2 Check that new password works.
+	client = s.getUserClient(c, accessKey, newSecretKey, "")
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("user could not create bucket: %v", err)
+	}
+
 	// 4. Check that user can be disabled and verify it.
 	err = s.adm.SetUserStatus(ctx, accessKey, madmin.AccountDisabled)
 	if err != nil {
@@ -260,6 +288,108 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	}
 }
 
+func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// 2. Create a user, associate policy and verify access
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+	// 2.1 check that user does not have any access to the bucket
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	// 2.2 create and associate policy to user
+	policy := "mypolicy-test-user-update"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+	// 2.3 check user has access to bucket
+	c.mustListObjects(ctx, uClient, bucket)
+	// 2.3 check that user cannot delete the bucket
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil || err.Error() != "Access Denied." {
+		c.Fatalf("bucket was deleted unexpectedly or got unexpected err: %v", err)
+	}
+
+	// 3. Craft a request to update the user's permissions
+	ep := s.adm.GetEndpointURL()
+	urlValue := url.Values{}
+	urlValue.Add("accessKey", accessKey)
+	u, err := url.Parse(fmt.Sprintf("%s://%s/minio/admin/v3/add-user?%s", ep.Scheme, ep.Host, s3utils.QueryEncode(urlValue)))
+	if err != nil {
+		c.Fatalf("unexpected url parse err: %v", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), nil)
+	if err != nil {
+		c.Fatalf("unexpected new request err: %v", err)
+	}
+	reqBodyArg := madmin.UserInfo{
+		SecretKey:  secretKey,
+		PolicyName: "consoleAdmin",
+		Status:     madmin.AccountEnabled,
+	}
+	buf, err := json.Marshal(reqBodyArg)
+	if err != nil {
+		c.Fatalf("unexpected json encode err: %v", err)
+	}
+	buf, err = madmin.EncryptData(secretKey, buf)
+	if err != nil {
+		c.Fatalf("unexpected encryption err: %v", err)
+	}
+
+	req.ContentLength = int64(len(buf))
+	sum := sha256.Sum256(buf)
+	req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
+	req.Body = ioutil.NopCloser(bytes.NewReader(buf))
+	req = signer.SignV4(*req, accessKey, secretKey, "", "")
+
+	// 3.1 Execute the request.
+	resp, err := s.TestSuiteCommon.client.Do(req)
+	if err != nil {
+		c.Fatalf("unexpected request err: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		c.Fatalf("got unexpected response: %#v\n", resp)
+	}
+
+	// 3.2 check that user cannot delete the bucket
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil || err.Error() != "Access Denied." {
+		c.Fatalf("User was able to escalate privileges (Err=%v)!", err)
+	}
+}
+
 func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()

cmd/admin-handlers.go

@@ -1869,6 +1869,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 
 			tls := getTLSInfo()
+			isK8s := IsKubernetes()
+			isDocker := IsDocker()
 			healthInfo.Minio.Info = madmin.MinioInfo{
 				Mode:         infoMessage.Mode,
 				Domain:       infoMessage.Domain,
@@ -1882,6 +1884,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				Backend:      infoMessage.Backend,
 				Servers:      servers,
 				TLS:          &tls,
+				IsKubernetes: &isK8s,
+				IsDocker:     &isDocker,
 			}
 			partialWrite(healthInfo)
 		}

cmd/admin-heal-ops.go

@@ -710,6 +710,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 
 	select {
 	case globalBackgroundHealRoutine.tasks <- task:
+		if serverDebugLog {
+			logger.Info("Task in the queue: %#v", task)
+		}
 	case <-h.ctx.Done():
 		return nil
 	}
@@ -885,6 +888,7 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
+		opts:      &h.settings,
 	}, madmin.HealItemObject)
 
 	// Wait and proceed if there are active requests

cmd/admin-router.go

@@ -194,11 +194,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalJoin)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateIAMItem)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateBucketItem)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalGetIDPSettings)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
 		}
 
 		if globalIsDistErasure {

cmd/api-errors.go

@@ -82,6 +82,7 @@ const (
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
+	ErrAccessKeyDisabled
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -129,6 +130,7 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
+	ErrReplicationDenyEditError
 	ErrReplicationNoMatchingRuleError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -513,6 +515,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrAccessKeyDisabled: {
+		Code:           "InvalidAccessKeyId",
+		Description:    "Your account is disabled; please contact your administrator.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -680,7 +687,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
-		Description:    "All access to this bucket has been disabled.",
+		Description:    "All access to this resource has been disabled.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedPolicy: {
@@ -888,6 +895,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching replication rule found for this object prefix",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationDenyEditError: {
+		Code:           "XMinioReplicationDenyEdit",
+		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",

cmd/auth-handler.go

@@ -238,20 +238,6 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
 
-	// If LDAP claim key is set, return here.
-	if _, ok := claims.MapClaims[ldapUser]; ok {
-		return claims.Map(), nil
-	}
-
-	// Session token must have a policy, reject requests without policy
-	// claim.
-	_, pokOpenIDClaimName := claims.MapClaims[iamPolicyClaimNameOpenID()]
-	_, pokOpenIDRoleArn := claims.MapClaims[roleArnClaim]
-	_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
-	if !pokOpenIDClaimName && !pokOpenIDRoleArn && !pokSA {
-		return nil, errAuthentication
-	}
-
 	return claims.Map(), nil
 }
 

cmd/background-newdisks-heal-ops.go

@@ -375,15 +375,13 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 
 			buckets, _ := z.ListBuckets(ctx)
 
-			buckets = append(buckets, BucketInfo{
-				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			})
-
 			// Buckets data are dispersed in multiple zones/sets, make
 			// sure to heal all bucket metadata configuration.
-			buckets = append(buckets, []BucketInfo{
-				{Name: pathJoin(minioMetaBucket, bucketMetaPrefix)},
-			}...)
+			buckets = append(buckets, BucketInfo{
+				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+			}, BucketInfo{
+				Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+			})
 
 			// Heal latest buckets first.
 			sort.Slice(buckets, func(i, j int) bool {
@@ -441,10 +439,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								continue
 							}
 
-							logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
+							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
+								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
 							logger.Info("Summary:\n")
 							tracker.printTo(os.Stdout)
 							logger.LogIf(ctx, tracker.delete(ctx))
+							logger.Info("\n")
 
 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())

cmd/bootstrap-peer-server.go

@@ -86,14 +86,28 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 		}
 	}
 	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
-		return fmt.Errorf("Expected same MINIO_ environment variables and values")
+		var missing []string
+		var mismatching []string
+		for k, v := range s1.MinioEnv {
+			ev, ok := s2.MinioEnv[k]
+			if !ok {
+				missing = append(missing, k)
+			} else if v != ev {
+				mismatching = append(mismatching, k)
+			}
+		}
+		if len(mismatching) > 0 {
+			return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
+		}
+		return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
 	}
 	return nil
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":        {},
-	"MINIO_CERT_PASSWD": {},
+	"MINIO_OPTS":         {},
+	"MINIO_CERT_PASSWD":  {},
+	"MINIO_SERVER_DEBUG": {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/bucket-handlers.go

@@ -1280,7 +1280,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 		if globalDNSConfig != nil {
 			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, pl1ease fix it manually", err2))
+				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
 			}
 		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
@@ -1586,7 +1586,10 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return
@@ -1688,6 +1691,10 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if err := globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle-handlers.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/bucket/object/lock"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
@@ -80,17 +79,6 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Disallow MaxNoncurrentVersions if bucket has object locking enabled
-	var rCfg lock.Retention
-	if rCfg, err = globalBucketObjectLockSys.Get(bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if rCfg.LockEnabled && bucketLifecycle.HasMaxNoncurrentVersions() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidLifecycleWithObjectLock), r.URL)
-		return
-	}
-
 	// Validate the transition storage ARNs
 	if err = validateTransitionTier(bucketLifecycle); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/bucket-lifecycle.go

@@ -81,21 +81,21 @@ type expiryTask struct {
 }
 
 type expiryState struct {
-	once              sync.Once
-	byDaysCh          chan expiryTask
-	byMaxNoncurrentCh chan maxNoncurrentTask
+	once                sync.Once
+	byDaysCh            chan expiryTask
+	byNewerNoncurrentCh chan newerNoncurrentTask
 }
 
 // PendingTasks returns the number of pending ILM expiry tasks.
 func (es *expiryState) PendingTasks() int {
-	return len(es.byDaysCh) + len(es.byMaxNoncurrentCh)
+	return len(es.byDaysCh) + len(es.byNewerNoncurrentCh)
 }
 
 // close closes work channels exactly once.
 func (es *expiryState) close() {
 	es.once.Do(func() {
 		close(es.byDaysCh)
-		close(es.byMaxNoncurrentCh)
+		close(es.byNewerNoncurrentCh)
 	})
 }
 
@@ -109,13 +109,13 @@ func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersi
 	}
 }
 
-// enqueueByMaxNoncurrent enqueues object versions expired by
-// MaxNoncurrentVersions limit for expiry.
-func (es *expiryState) enqueueByMaxNoncurrent(bucket string, versions []ObjectToDelete) {
+// enqueueByNewerNoncurrent enqueues object versions expired by
+// NewerNoncurrentVersions limit for expiry.
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byMaxNoncurrentCh <- maxNoncurrentTask{bucket: bucket, versions: versions}:
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
 	default:
 	}
 }
@@ -124,8 +124,8 @@ var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:          make(chan expiryTask, 10000),
-		byMaxNoncurrentCh: make(chan maxNoncurrentTask, 10000),
+		byDaysCh:            make(chan expiryTask, 10000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
 	}
 }
 
@@ -141,15 +141,15 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 		}
 	}()
 	go func() {
-		for t := range globalExpiryState.byMaxNoncurrentCh {
+		for t := range globalExpiryState.byNewerNoncurrentCh {
 			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
 		}
 	}()
 }
 
-// maxNoncurrentTask encapsulates arguments required by worker to expire objects
-// by MaxNoncurrentVersions
-type maxNoncurrentTask struct {
+// newerNoncurrentTask encapsulates arguments required by worker to expire objects
+// by NewerNoncurrentVersions
+type newerNoncurrentTask struct {
 	bucket   string
 	versions []ObjectToDelete
 }

cmd/bucket-replication-stats.go

@@ -51,6 +51,9 @@ func (r *ReplicationStats) Delete(bucket string) {
 	r.Lock()
 	defer r.Unlock()
 	delete(r.Cache, bucket)
+
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
 	delete(r.UsageCache, bucket)
 
 }
@@ -82,10 +85,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 	bs, ok := r.Cache[bucket]
 	if !ok {
 		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		r.Cache[bucket] = bs
 	}
 	b, ok := bs.Stats[arn]
 	if !ok {
 		b = &BucketReplicationStat{}
+		bs.Stats[arn] = b
 	}
 	switch status {
 	case replication.Completed:
@@ -115,9 +120,6 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 			b.ReplicaSize += n
 		}
 	}
-
-	bs.Stats[arn] = b
-	r.Cache[bucket] = bs
 }
 
 // GetInitialUsage get replication metrics available at the time of cluster initialization
@@ -128,10 +130,10 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	r.ulock.RLock()
 	defer r.ulock.RUnlock()
 	st, ok := r.UsageCache[bucket]
-	if ok {
-		return st.Clone()
+	if !ok {
+		return BucketReplicationStats{}
 	}
-	return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+	return st.Clone()
 }
 
 // Get replication metrics for a bucket from this node since this node came up.
@@ -145,7 +147,7 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 
 	st, ok := r.Cache[bucket]
 	if !ok {
-		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		return BucketReplicationStats{}
 	}
 	return st.Clone()
 }
@@ -162,41 +164,46 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
 	rTimer := time.NewTimer(time.Minute * 1)
 	defer rTimer.Stop()
+	var (
+		dui DataUsageInfo
+		err error
+	)
+outer:
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-rTimer.C:
-			dui, err := loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
+			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
 			if err != nil {
 				continue
 			}
-			// data usage has not captured any data yet.
-			if dui.LastUpdate.IsZero() {
-				continue
+			// If LastUpdate is set, data usage is available.
+			if !dui.LastUpdate.IsZero() {
+				break outer
 			}
-			m := make(map[string]*BucketReplicationStats)
-			for bucket, usage := range dui.BucketsUsage {
-				b := &BucketReplicationStats{
-					Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
-				}
-				for arn, uinfo := range usage.ReplicationInfo {
-					b.Stats[arn] = &BucketReplicationStat{
-						FailedSize:     int64(uinfo.ReplicationFailedSize),
-						ReplicatedSize: int64(uinfo.ReplicatedSize),
-						ReplicaSize:    int64(uinfo.ReplicaSize),
-						FailedCount:    int64(uinfo.ReplicationFailedCount),
-					}
-				}
-				b.ReplicaSize += int64(usage.ReplicaSize)
-				if b.hasReplicationUsage() {
-					m[bucket] = b
-				}
-			}
-			r.ulock.Lock()
-			defer r.ulock.Unlock()
-			r.UsageCache = m
-			return
 		}
 	}
+
+	m := make(map[string]*BucketReplicationStats)
+	for bucket, usage := range dui.BucketsUsage {
+		b := &BucketReplicationStats{
+			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
+		}
+		for arn, uinfo := range usage.ReplicationInfo {
+			b.Stats[arn] = &BucketReplicationStat{
+				FailedSize:     int64(uinfo.ReplicationFailedSize),
+				ReplicatedSize: int64(uinfo.ReplicatedSize),
+				ReplicaSize:    int64(uinfo.ReplicaSize),
+				FailedCount:    int64(uinfo.ReplicationFailedCount),
+			}
+		}
+		b.ReplicaSize += int64(usage.ReplicaSize)
+		if b.hasReplicationUsage() {
+			m[bucket] = b
+		}
+	}
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
+	r.UsageCache = m
 }

cmd/bucket-replication.go

@@ -62,6 +62,8 @@ const (
 	ObjectLockRetentionTimestamp = "objectlock-retention-timestamp"
 	// ObjectLockLegalHoldTimestamp - the last time a legal hold metadata modification happened on this cluster for this object version
 	ObjectLockLegalHoldTimestamp = "objectlock-legalhold-timestamp"
+	// ReplicationWorkerMultiplier is suggested worker multiplier if traffic exceeds replication worker capacity
+	ReplicationWorkerMultiplier = 1.5
 )
 
 // gets replication config associated to a given bucket name.
@@ -515,8 +517,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		}
 		return
 	}
-	// early return if already replicated delete marker for existing object replication
-	if dobj.DeleteMarkerVersionID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
+	// early return if already replicated delete marker for existing object replication/ healing delete markers
+	if dobj.DeleteMarkerVersionID != "" && (dobj.OpType == replication.ExistingObjectReplicationType || dobj.OpType == replication.HealReplicationType) {
 		if _, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, miniogo.StatObjectOptions{
 			VersionID: versionID,
 			Internal: miniogo.AdvancedGetOptions{
@@ -524,10 +526,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 			}}); isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
 			if dobj.VersionID == "" {
 				rinfo.ReplicationStatus = replication.Completed
-			} else {
-				rinfo.VersionPurgeStatus = Complete
+				return
 			}
-			return
 		}
 	}
 
@@ -1412,6 +1412,14 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 	}
 }
 
+// suggestedWorkers recommends an increase in number of workers to meet replication load.
+func (p *ReplicationPool) suggestedWorkers(failQueue bool) int {
+	if failQueue {
+		return int(float64(p.mrfWorkerSize) * ReplicationWorkerMultiplier)
+	}
+	return int(float64(p.workerSize) * ReplicationWorkerMultiplier)
+}
+
 func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
@@ -1425,6 +1433,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication failed workers could not keep up with healing failures - consider increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
 	}
 }
 
@@ -1450,6 +1459,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1486,6 +1496,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1797,22 +1808,15 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	// add initial usage stat to cluster stats
 	usageStat := globalReplicationStats.GetInitialUsage(bucket)
 	totReplicaSize += usageStat.ReplicaSize
-	if usageStat.Stats != nil {
-		for arn, stat := range usageStat.Stats {
-			st := stats[arn]
-			if st == nil {
-				st = &BucketReplicationStat{
-					ReplicatedSize: stat.ReplicatedSize,
-					FailedSize:     stat.FailedSize,
-					FailedCount:    stat.FailedCount,
-				}
-			} else {
-				st.ReplicatedSize += stat.ReplicatedSize
-				st.FailedSize += stat.FailedSize
-				st.FailedCount += stat.FailedCount
-			}
+	for arn, stat := range usageStat.Stats {
+		st, ok := stats[arn]
+		if !ok {
+			st = &BucketReplicationStat{}
 			stats[arn] = st
 		}
+		st.ReplicatedSize += stat.ReplicatedSize
+		st.FailedSize += stat.FailedSize
+		st.FailedCount += stat.FailedCount
 	}
 	s = BucketReplicationStats{
 		Stats: make(map[string]*BucketReplicationStat, len(stats)),

cmd/bucket-stats.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"sync/atomic"
 	"time"
 )
 
@@ -52,13 +51,6 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 	rl.UploadHistogram.Add(size, duration)
 }
 
-// Clone replication latency
-func (rl ReplicationLatency) clone() ReplicationLatency {
-	return ReplicationLatency{
-		UploadHistogram: rl.UploadHistogram.Clone(),
-	}
-}
-
 // BucketStats bucket statistics
 type BucketStats struct {
 	ReplicationStats BucketReplicationStats
@@ -88,29 +80,18 @@ func (brs *BucketReplicationStats) Empty() bool {
 }
 
 // Clone creates a new BucketReplicationStats copy
-func (brs BucketReplicationStats) Clone() BucketReplicationStats {
-	c := BucketReplicationStats{
-		Stats: make(map[string]*BucketReplicationStat, len(brs.Stats)),
-	}
-	// This is called only by replicationStats cache and already holds a read lock before calling Clone()
+func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
+	// This is called only by replicationStats cache and already holds a
+	// read lock before calling Clone()
+
+	c = brs
+	// We need to copy the map, so we do not reference the one in `brs`.
+	c.Stats = make(map[string]*BucketReplicationStat, len(brs.Stats))
 	for arn, st := range brs.Stats {
-		c.Stats[arn] = &BucketReplicationStat{
-			FailedSize:     atomic.LoadInt64(&st.FailedSize),
-			ReplicatedSize: atomic.LoadInt64(&st.ReplicatedSize),
-			ReplicaSize:    atomic.LoadInt64(&st.ReplicaSize),
-			FailedCount:    atomic.LoadInt64(&st.FailedCount),
-			PendingSize:    atomic.LoadInt64(&st.PendingSize),
-			PendingCount:   atomic.LoadInt64(&st.PendingCount),
-			Latency:        st.Latency.clone(),
-		}
+		// make a copy of `*st`
+		s := *st
+		c.Stats[arn] = &s
 	}
-	// update total counts across targets
-	c.FailedSize = atomic.LoadInt64(&brs.FailedSize)
-	c.FailedCount = atomic.LoadInt64(&brs.FailedCount)
-	c.PendingCount = atomic.LoadInt64(&brs.PendingCount)
-	c.PendingSize = atomic.LoadInt64(&brs.PendingSize)
-	c.ReplicaSize = atomic.LoadInt64(&brs.ReplicaSize)
-	c.ReplicatedSize = atomic.LoadInt64(&brs.ReplicatedSize)
 	return c
 }
 

cmd/bucket-versioning-handler.go

@@ -63,6 +63,15 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL)
+		return
+	}
+
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",

cmd/common-main.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -65,6 +68,7 @@ import (
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+var shardDiskTimeDelta time.Duration
 var defaultAWSCredProvider []credentials.Provider
 
 func init() {
@@ -134,6 +138,12 @@ func init() {
 		},
 	}
 
+	var err error
+	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
+	if err != nil {
+		shardDiskTimeDelta = 1 * time.Minute
+	}
+
 	// All minio-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
@@ -307,7 +317,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
+	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -433,7 +443,166 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
 }
 
+type envKV struct {
+	Key   string
+	Value string
+	Skip  bool
+}
+
+func (e envKV) String() string {
+	if e.Skip {
+		return ""
+	}
+	return fmt.Sprintf("%s=%s", e.Key, e.Value)
+}
+
+func parsEnvEntry(envEntry string) (envKV, error) {
+	envEntry = strings.TrimSpace(envEntry)
+	if envEntry == "" {
+		// Skip all empty lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+	const envSeparator = "="
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	if len(envTokens) != 2 {
+		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
+	}
+
+	key := envTokens[0]
+	val := envTokens[1]
+
+	// Remove quotes from the value if found
+	if len(val) >= 2 {
+		quote := val[0]
+		if (quote == '"' || quote == '\'') && val[len(val)-1] == quote {
+			val = val[1 : len(val)-1]
+		}
+	}
+
+	return envKV{
+		Key:   key,
+		Value: val,
+	}, nil
+}
+
+// Similar to os.Environ returns a copy of strings representing
+// the environment values from a file, in the form "key, value".
+// in a structured form.
+func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
+	f, err := os.Open(envConfigFile)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	var ekvs []envKV
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		ekv, err := parsEnvEntry(scanner.Text())
+		if err != nil {
+			return nil, err
+		}
+		if ekv.Skip {
+			// Skips empty lines
+			continue
+		}
+		ekvs = append(ekvs, ekv)
+	}
+	if err = scanner.Err(); err != nil {
+		return nil, err
+	}
+	return ekvs, nil
+}
+
+func readFromSecret(sp string) (string, error) {
+	// Supports reading path from docker secrets, filename is
+	// relative to /run/secrets/ position.
+	if isFile(pathJoin("/run/secrets/", sp)) {
+		sp = pathJoin("/run/secrets/", sp)
+	}
+	credBuf, err := ioutil.ReadFile(sp)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return "", nil
+		}
+		return "", err
+	}
+	return string(bytes.TrimSpace(credBuf)), nil
+}
+
+func loadEnvVarsFromFiles() {
+	if env.IsSet(config.EnvAccessKeyFile) {
+		accessKey, err := readFromSecret(env.Get(config.EnvAccessKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if accessKey != "" {
+			os.Setenv(config.EnvRootUser, accessKey)
+		}
+	}
+
+	if env.IsSet(config.EnvSecretKeyFile) {
+		secretKey, err := readFromSecret(env.Get(config.EnvSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if secretKey != "" {
+			os.Setenv(config.EnvRootPassword, secretKey)
+		}
+	}
+
+	if env.IsSet(config.EnvRootUserFile) {
+		rootUser, err := readFromSecret(env.Get(config.EnvRootUserFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootUser != "" {
+			os.Setenv(config.EnvRootUser, rootUser)
+		}
+	}
+
+	if env.IsSet(config.EnvRootPasswordFile) {
+		rootPassword, err := readFromSecret(env.Get(config.EnvRootPasswordFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootPassword != "" {
+			os.Setenv(config.EnvRootPassword, rootPassword)
+		}
+	}
+
+	if env.IsSet(config.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
+		}
+		if kmsSecret != "" {
+			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+		}
+	}
+
+	if env.IsSet(config.EnvConfigEnvFile) {
+		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the config environment file")
+		}
+		for _, ekv := range ekvs {
+			os.Setenv(ekv.Key, ekv.Value)
+		}
+	}
+}
+
 func handleCommonEnvVars() {
+	loadEnvVarsFromFiles()
+
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -530,8 +699,8 @@ func handleCommonEnvVars() {
 	// in-place update is off.
 	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff)
 
-	// Check if the supported credential env vars, "MINIO_ROOT_USER" and
-	// "MINIO_ROOT_PASSWORD" are provided
+	// Check if the supported credential env vars,
+	// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
 	// Warn user if deprecated environment variables,
 	// "MINIO_ACCESS_KEY" and "MINIO_SECRET_KEY", are defined
 	// Check all error conditions first
@@ -552,25 +721,24 @@ func handleCommonEnvVars() {
 	// are defined or both are not defined.
 	// Check both cases and authenticate them if correctly defined
 	var user, password string
-	haveRootCredentials := false
-	haveAccessCredentials := false
+	var hasCredentials bool
 	//nolint:gocritic
 	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		user = env.Get(config.EnvRootUser, "")
 		password = env.Get(config.EnvRootPassword, "")
-		haveRootCredentials = true
+		hasCredentials = true
 	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
-		haveAccessCredentials = true
+		hasCredentials = true
 	}
-	if haveRootCredentials || haveAccessCredentials {
+	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
 			logger.Fatal(config.ErrInvalidCredentials(err),
 				"Unable to validate credentials inherited from the shell environment")
 		}
-		if haveAccessCredentials {
+		if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,

cmd/common-main_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"reflect"
+	"testing"
+)
+
+func Test_readFromSecret(t *testing.T) {
+	testCases := []struct {
+		content       string
+		expectedErr   bool
+		expectedValue string
+	}{
+		{
+			"value\n",
+			false,
+			"value",
+		},
+		{
+			" \t\n Hello, Gophers \n\t\r\n",
+			false,
+			"Hello, Gophers",
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			value, err := readFromSecret(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if value != testCase.expectedValue {
+				t.Errorf("Expected %s, got %s", testCase.expectedValue, value)
+			}
+		})
+	}
+}
+
+func Test_minioEnvironFromFile(t *testing.T) {
+	testCases := []struct {
+		content      string
+		expectedErr  bool
+		expectedEkvs []envKV
+	}{
+		{`
+export MINIO_ROOT_USER=minio
+export MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		// Value with double quotes
+		{`export MINIO_ROOT_USER="minio"`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		// Value with single quotes
+		{`export MINIO_ROOT_USER='minio'`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		{`
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		{`
+export MINIO_ROOT_USERminio
+export MINIO_ROOT_PASSWORD=minio123`,
+			true,
+			nil,
+		},
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			ekvs, err := minioEnvironFromFile(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) {
+				t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs)
+			}
+		})
+	}
+}

cmd/config-common.go

@@ -29,26 +29,31 @@ import (
 
 var errConfigNotFound = errors.New("config file not found")
 
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+func readConfigWithMetadata(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, ObjectInfo, error) {
 	r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
-			return nil, errConfigNotFound
+			return nil, ObjectInfo{}, errConfigNotFound
 		}
 
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	defer r.Close()
 
 	buf, err := ioutil.ReadAll(r)
 	if err != nil {
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	if len(buf) == 0 {
-		return nil, errConfigNotFound
+		return nil, ObjectInfo{}, errConfigNotFound
 	}
-	return buf, nil
+	return buf, r.ObjInfo, nil
+}
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	buf, _, err := readConfigWithMetadata(ctx, objAPI, configFile)
+	return buf, err
 }
 
 type objectDeleter interface {

cmd/config-current.go

@@ -62,7 +62,7 @@ func initHelp() {
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
 		config.CredentialsSubSys:    config.DefaultCredentialKVS,
-		config.LoggerWebhookSubSys:  logger.DefaultKVS,
+		config.LoggerWebhookSubSys:  logger.DefaultLoggerWebhookKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
@@ -543,7 +543,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	loggerCfg, err := logger.LookupConfig(s)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger/audit targets: %w", err))
 	}
 
 	for _, l := range loggerCfg.HTTP {
@@ -553,7 +553,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http logging
 			if err = logger.AddTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server logger HTTP target: %w", err))
 			}
 		}
 	}
@@ -565,7 +565,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http audit logging
 			if err = logger.AddAuditTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit HTTP target: %w", err))
 			}
 		}
 	}
@@ -575,7 +575,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.LogOnce = logger.LogOnceIf
 			// Enable Kafka audit logging
 			if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit Kafka target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit Kafka target: %w", err))
 			}
 		}
 	}

cmd/data-scanner.go

@@ -71,7 +71,17 @@ var (
 
 // initDataScanner will start the scanner in the background.
 func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
-	go runDataScanner(ctx, objAPI)
+	go func() {
+		// Run the data scanner in a loop
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				runDataScanner(ctx, objAPI)
+			}
+		}
+	}()
 }
 
 type safeDuration struct {
@@ -972,14 +982,14 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 
 }
 
-// applyMaxNoncurrentVersionLimit removes noncurrent versions older than the most recent MaxNoncurrentVersions configured.
+// applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
 	if i.lifeCycle == nil {
 		return fivs, nil
 	}
 
-	lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
 	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
 		return fivs, nil
 	}
@@ -992,6 +1002,7 @@ func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o Obje
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
 		obj := fi.ToObjectInfo(i.bucket, i.objectPath())
+		// skip versions with object locking enabled
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			if i.debug {
 				if obj.VersionID != "" {
@@ -1000,22 +1011,34 @@ func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o Obje
 					console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 				}
 			}
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
 			continue
 		}
+
+		// NoncurrentDays not passed yet.
+		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, days)) {
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
+			continue
+		}
+
 		toDel = append(toDel, ObjectToDelete{
 			ObjectName: fi.Name,
 			VersionID:  fi.VersionID,
 		})
 	}
 
-	globalExpiryState.enqueueByMaxNoncurrent(i.bucket, toDel)
+	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
 	return fivs, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
 func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
-	return i.applyMaxNoncurrentVersionLimit(ctx, o, fivs)
+	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1246,6 +1269,7 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				DeleteMarker:          roi.DeleteMarker,
 			},
 			Bucket: roi.Bucket,
+			OpType: replication.HealReplicationType,
 		}
 		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType

cmd/disk-cache.go

@@ -706,7 +706,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 				oi, _, err := dcache.Stat(GlobalContext, bucket, object)
 				// avoid cache overwrite if another background routine filled cache
 				if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, true)
+					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, false)
 				}
 			}()
 		}

cmd/erasure-bucket.go

@@ -64,8 +64,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		}, index)
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, writeQuorum)
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
 	return toObjectErr(err, bucket)
 }
 
@@ -121,8 +120,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 	// reduce to one error based on read quorum.
 	// `nil` is deliberately passed for ignoredErrs
 	// because these errors were already ignored.
-	readQuorum := getReadQuorum(len(storageDisks))
-	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, readQuorum)
+	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, er.defaultRQuorum())
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
@@ -167,8 +165,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts D
 		return nil
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
+	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, er.defaultWQuorum())
 	if err == errErasureWriteQuorum && !opts.NoRecreate {
 		undoDeleteBucket(storageDisks, bucket)
 	}

cmd/erasure-healing-common.go

@@ -26,29 +26,59 @@ import (
 )
 
 // commonTime returns a maximally occurring time from a list of time.
-func commonTime(modTimes []time.Time) (modTime time.Time) {
-	timeOccurenceMap := make(map[int64]int, len(modTimes))
+func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) {
+	timeOccurenceMap := make(map[int64]int, len(times))
+	groupNano := group.Nanoseconds()
 	// Ignore the uuid sentinel and count the rest.
-	for _, t := range modTimes {
+	for _, t := range times {
 		if t.Equal(timeSentinel) {
 			continue
 		}
-		timeOccurenceMap[t.UnixNano()]++
+		nano := t.UnixNano()
+		if group > 0 {
+			for k := range timeOccurenceMap {
+				if k == nano {
+					// We add to ourself later
+					continue
+				}
+				diff := k - nano
+				if diff < 0 {
+					diff = -diff
+				}
+				// We are within the limit
+				if diff < groupNano {
+					timeOccurenceMap[k]++
+				}
+			}
+		}
+		// Add ourself...
+		timeOccurenceMap[nano]++
 	}
 
-	var maxima int // Counter for remembering max occurrence of elements.
+	maxima = 0 // Counter for remembering max occurrence of elements.
+	latest := int64(0)
 
 	// Find the common cardinality from previously collected
 	// occurrences of elements.
 	for nano, count := range timeOccurenceMap {
-		t := time.Unix(0, nano).UTC()
-		if count > maxima || (count == maxima && t.After(modTime)) {
+		if count < maxima {
+			continue
+		}
+
+		// We are at or above maxima
+		if count > maxima || nano > latest {
 			maxima = count
-			modTime = t
+			latest = nano
 		}
 	}
 
-	// Return the collected common modTime.
+	// Return the collected common max time, with maxima
+	return time.Unix(0, latest).UTC(), maxima
+}
+
+// commonTime returns a maximally occurring time from a list of time.
+func commonTime(modTimes []time.Time) (modTime time.Time) {
+	modTime, _ = commonTimeAndOccurence(modTimes, 0)
 	return modTime
 }
 
@@ -88,6 +118,19 @@ func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks
 	}
 }
 
+// Extracts list of disk mtimes from FileInfo slice and returns, skips
+// slice elements that have errors.
+func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
+	diskMTimes = bootModtimes(len(partsMetadata))
+	for index, metadata := range partsMetadata {
+		if metadata.IsValid() {
+			// Once the file is found, save the disk mtime saved on disk.
+			diskMTimes[index] = metadata.DiskMTime
+		}
+	}
+	return diskMTimes
+}
+
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -182,8 +225,15 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
-	errs []error, latestMeta FileInfo,
-	bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
+	errs []error, latestMeta FileInfo, bucket, object string,
+	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time) {
+
+	var diskMTime time.Time
+	var shardFix bool
+	if !latestMeta.DataShardFixed() {
+		diskMTime = pickValidDiskTimeWithQuorum(partsMetadata,
+			latestMeta.Erasure.DataBlocks)
+	}
 
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
@@ -234,6 +284,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 
 		if erasureDistributionReliable {
 			if !meta.IsValid() {
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
 				continue
 			}
 
@@ -256,6 +308,23 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			}
 		}
 
+		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+			if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
+				// not with in acceptable delta, skip.
+				// If disk mTime mismatches it is considered outdated
+				// https://github.com/minio/minio/pull/13803
+				//
+				// This check only is active if we could find maximally
+				// occurring disk mtimes that are somewhat same across
+				// the quorum. Allowing to skip those shards which we
+				// might think are wrong.
+				shardFix = true
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+		}
+
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
@@ -298,5 +367,10 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	return availableDisks, dataErrs
+	if shardFix {
+		// Only when shard is fixed return an appropriate disk mtime value.
+		return availableDisks, dataErrs, diskMTime
+	} // else return timeSentinel for disk mtime
+
+	return availableDisks, dataErrs, timeSentinel
 }

cmd/erasure-healing-common_test.go

@@ -251,7 +251,8 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
 			}
-			availableDisks, newErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan)
+			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, bucket, object, madmin.HealDeepScan)
 			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
@@ -433,7 +434,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					test.expectedTime, modTime)
 			}
 
-			availableDisks, newErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan)
+			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, bucket, object, madmin.HealDeepScan)
 			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
@@ -494,7 +496,8 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	erasureDisks, _ = listOnlineDisks(erasureDisks, partsMetadata, errs)
 
-	filteredDisks, errs := disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -515,7 +518,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour)
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -535,7 +539,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[1].DataDir = "foo-random"
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -571,7 +576,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, errs = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, errs, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))

cmd/erasure-healing.go

@@ -24,12 +24,38 @@ import (
 	"fmt"
 	"io"
 	"sync"
+	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 )
 
+const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
+
+// AcceptableDelta returns 'true' if the fi.DiskMTime is under
+// acceptable delta of "delta" duration with maxTime.
+//
+// This code is primarily used for heuristic detection of
+// incorrect shards, as per https://github.com/minio/minio/pull/13803
+//
+// This check only is active if we could find maximally
+// occurring disk mtimes that are somewhat same across
+// the quorum. Allowing to skip those shards which we
+// might think are wrong.
+func (fi FileInfo) AcceptableDelta(maxTime time.Time, delta time.Duration) bool {
+	diff := maxTime.Sub(fi.DiskMTime)
+	if diff < 0 {
+		diff = -diff
+	}
+	return diff < delta
+}
+
+// DataShardFixed - data shard fixed?
+func (fi FileInfo) DataShardFixed() bool {
+	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
+}
+
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -224,6 +250,11 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 		return true
 	}
 	if erErr == nil {
+		if meta.XLV1 {
+			// Legacy means heal always
+			// always check first.
+			return true
+		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -234,19 +265,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 				return true
 			}
 		}
-		if !latestMeta.MetadataEquals(meta) {
-			return true
-		}
-		if !latestMeta.TransitionInfoEquals(meta) {
-			return true
-		}
-		if !latestMeta.ReplicationInfoEquals(meta) {
-			return true
-		}
-		if !latestMeta.ModTime.Equal(meta.ModTime) {
-			return true
-		}
-		if meta.XLV1 {
+		if !latestMeta.Equals(meta) {
 			return true
 		}
 	}
@@ -267,12 +286,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Initialize heal result object
 	result = madmin.HealResultItem{
-		Type:         madmin.HealItemObject,
-		Bucket:       bucket,
-		Object:       object,
-		DiskCount:    len(storageDisks),
-		ParityBlocks: er.defaultParityCount,
-		DataBlocks:   len(storageDisks) - er.defaultParityCount,
+		Type:      madmin.HealItemObject,
+		Bucket:    bucket,
+		Object:    object,
+		DiskCount: len(storageDisks),
 	}
 
 	if !opts.NoLock {
@@ -287,18 +304,27 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
+	if isAllNotFound(errs) {
+		// Nothing to do, file is already gone.
+		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), nil
+	}
 
-	if _, err = getLatestFileInfo(ctx, partsMetadata, errs); err != nil {
+	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
+	if err != nil {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
 	}
 
+	result.ParityBlocks = result.DiskCount - readQuorum
+	result.DataBlocks = readQuorum
+
 	// List of disks having latest version of the object er.meta
 	// (by modtime).
 	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, result.DataBlocks)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, readQuorum)
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, versionID)
 	}
@@ -312,7 +338,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
 	// Loop to find number of disks with valid data, per-drive
@@ -373,13 +399,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	}
 
 	if isAllNotFound(errs) {
-		err = toObjectErr(errFileNotFound, bucket, object)
-		if versionID != "" {
-			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-		}
 		// File is fully gone, fileInfo is empty.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), err
+			bucket, object, versionID), nil
 	}
 
 	// If less than read quorum number of disks have all the parts
@@ -562,6 +584,20 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 	}
 
+	if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+		// Update metadata to indicate special fix.
+		_, err = er.PutObjectMetadata(ctx, bucket, object, ObjectOptions{
+			NoLock: true,
+			UserDefined: map[string]string{
+				reservedMetadataPrefixLowerDataShardFix: "true",
+				// another reserved metadata to capture original disk-mtime
+				// captured for this version of the object, to be used
+				// possibly in future to heal other versions if possible.
+				ReservedMetadataPrefixLower + "disk-mtime": diskMTime.String(),
+			},
+		})
+	}
+
 	// Set the size of the object in the heal result
 	result.ObjectSize = latestMeta.Size
 
@@ -630,7 +666,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 	}
 	if dryRun || danglingObject || isAllNotFound(errs) {
 		// Nothing to do, file is already gone.
-		return hr, toObjectErr(errFileNotFound, bucket, object)
+		return hr, nil
 	}
 	for i, err := range errs {
 		if err == errVolumeNotFound || err == errFileNotFound {
@@ -735,8 +771,15 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 // A 0 length slice will always return false.
 func isAllNotFound(errs []error) bool {
 	for _, err := range errs {
-		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) || errors.Is(err, errFileVersionNotFound) {
-			continue
+		if err != nil {
+			switch err.Error() {
+			case errFileNotFound.Error():
+				fallthrough
+			case errVolumeNotFound.Error():
+				fallthrough
+			case errFileVersionNotFound.Error():
+				continue
+			}
 		}
 		return false
 	}
@@ -921,13 +964,9 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	// This allows to quickly check if all is ok or all are missing.
 	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
 	if isAllNotFound(errs) {
-		err = toObjectErr(errFileNotFound, bucket, object)
-		if versionID != "" {
-			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-		}
 		// Nothing to do, file is already gone.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
-			errs, bucket, object, versionID), err
+			errs, bucket, object, versionID), nil
 	}
 
 	// Heal the object.

cmd/erasure-healing_test.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/sha256"
+	"errors"
 	"io"
 	"os"
 	"path"
@@ -93,16 +94,16 @@ func TestHealing(t *testing.T) {
 	}
 
 	// After heal the meta file should be as expected.
-	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
+	if !fileInfoPreHeal.Equals(fileInfoPostHeal) {
 		t.Fatal("HealObject failed")
 	}
 
-	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "er.meta"))
+	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.meta"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Write er.meta with different modtime to simulate the case where a disk had
+	// Write xl.meta with different modtime to simulate the case where a disk had
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal
 	fileInfoOutDated.ModTime = time.Now()
@@ -122,7 +123,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	// After heal the meta file should be as expected.
-	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
+	if !fileInfoPreHeal.Equals(fileInfoPostHeal) {
 		t.Fatal("HealObject failed")
 	}
 
@@ -328,6 +329,298 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 }
 
+func TestHealCorrectQuorum(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	cfgFile := pathJoin(bucketConfigPrefix, bucket, ".test.bin")
+	if err = saveConfig(ctx, objLayer, cfgFile, data); err != nil {
+		t.Fatal(err)
+	}
+
+	hopts := madmin.HealOpts{
+		DryRun:   false,
+		Remove:   true,
+		ScanMode: madmin.HealNormalScan,
+	}
+
+	// Test 1: Remove the object backend files from the first disk.
+	z := objLayer.(*erasureServerPools)
+	for _, set := range z.serverPools {
+		er := set.sets[0]
+		erasureDisks := er.getDisks()
+
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, it should heal the content properly.
+		_, err = objLayer.HealObject(ctx, bucket, object, "", hopts)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		if countErrs(errs, nil) != len(fileInfos) {
+			t.Fatal("Expected all xl.meta healed, but partial heal detected")
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, it should heal the content properly.
+		_, err = objLayer.HealObject(ctx, minioMetaBucket, cfgFile, "", hopts)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
+		if countErrs(errs, nil) != len(fileInfos) {
+			t.Fatal("Expected all xl.meta healed, but partial heal detected")
+		}
+	}
+}
+
+func TestHealObjectCorruptedPools(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	// Test 1: Remove the object backend files from the first disk.
+	z := objLayer.(*erasureServerPools)
+	for _, set := range z.serverPools {
+		er := set.sets[0]
+		erasureDisks := er.getDisks()
+		firstDisk := erasureDisks[0]
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		if err != nil {
+			t.Fatalf("Failed to delete a file - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+		if err != nil {
+			t.Fatalf("Failed to heal object - %v", err)
+		}
+
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+			t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+		}
+
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+		if err != nil {
+			t.Errorf("Failure during deleting part.1 - %v", err)
+		}
+
+		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+		if err != nil {
+			t.Errorf("Failure during creating part.1 - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			t.Errorf("Expected nil but received %v", err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if !reflect.DeepEqual(fi, nfi) {
+			t.Fatalf("FileInfo not equal after healing")
+		}
+
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+		if err != nil {
+			t.Errorf("Failure during deleting part.1 - %v", err)
+		}
+
+		bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
+		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+		if err != nil {
+			t.Errorf("Failure during creating part.1 - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			t.Errorf("Expected nil but received %v", err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if !reflect.DeepEqual(fi, nfi) {
+			t.Fatalf("FileInfo not equal after healing")
+		}
+
+		// Test 4: checks if HealObject returns an error when xl.meta is not found
+		// in more than read quorum number of disks, to create a corrupted situation.
+		for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, expect to receive errFileNotFound.
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			if _, ok := err.(ObjectNotFound); !ok {
+				t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+			}
+		}
+
+		// since majority of xl.meta's are not available, object should be successfully deleted.
+		_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+
+		for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
+			_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+			if err == nil {
+				t.Errorf("Expected xl.meta file to be not present, but succeeeded")
+			}
+		}
+	}
+}
+
 func TestHealObjectCorrupted(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -405,7 +698,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
 	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
@@ -549,7 +842,7 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
 	erasureDisks := er.getDisks()
@@ -565,7 +858,7 @@ func TestHealObjectErasure(t *testing.T) {
 
 	// Try healing now, expect to receive errDiskNotFound.
 	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
-	// since majority of er.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
+	// since majority of xl.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
 		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
 	}
@@ -663,6 +956,8 @@ func TestHealLastDataShard(t *testing.T) {
 		{"1MiB", 1 * humanize.MiByte},
 		{"5MiB", 5 * humanize.MiByte},
 		{"10MiB", 10 * humanize.MiByte},
+		{"5MiB-1KiB", 5*humanize.MiByte - 1*humanize.KiByte},
+		{"10MiB-1Kib", 10*humanize.MiByte - 1*humanize.KiByte},
 	}
 
 	for _, test := range tests {
@@ -697,7 +992,8 @@ func TestHealLastDataShard(t *testing.T) {
 				t.Fatalf("Failed to make a bucket - %v", err)
 			}
 
-			_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+			_, err = obj.PutObject(ctx, bucket, object,
+				mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 
 			if err != nil {
 				t.Fatal(err)
@@ -722,7 +1018,9 @@ func TestHealLastDataShard(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Failed to delete a file - %v", err)
 			}
-			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+				ScanMode: madmin.HealNormalScan,
+			})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -736,21 +1034,23 @@ func TestHealLastDataShard(t *testing.T) {
 			firstHealedH := sha256.New()
 			_, err = io.Copy(firstHealedH, firstGr)
 			if err != nil {
-				t.Fatal()
+				t.Fatal(err)
 			}
 			firstHealedDataSha256 := firstHealedH.Sum(nil)
 
 			if !bytes.Equal(actualSha256, firstHealedDataSha256) {
-				t.Fatal("object healed wrong")
+				t.Fatalf("object healed wrong, expected %x, got %x",
+					actualSha256, firstHealedDataSha256)
 			}
 
 			// remove another data shard
-			err = removeAll(pathJoin(shuffledDisks[1].String(), bucket, object))
-			if err != nil {
+			if err = removeAll(pathJoin(shuffledDisks[1].String(), bucket, object)); err != nil {
 				t.Fatalf("Failed to delete a file - %v", err)
 			}
 
-			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+				ScanMode: madmin.HealNormalScan,
+			})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -764,12 +1064,13 @@ func TestHealLastDataShard(t *testing.T) {
 			secondHealedH := sha256.New()
 			_, err = io.Copy(secondHealedH, secondGr)
 			if err != nil {
-				t.Fatal()
+				t.Fatal(err)
 			}
 			secondHealedDataSha256 := secondHealedH.Sum(nil)
 
 			if !bytes.Equal(actualSha256, secondHealedDataSha256) {
-				t.Fatal("object healed wrong")
+				t.Fatalf("object healed wrong, expected %x, got %x",
+					actualSha256, secondHealedDataSha256)
 			}
 		})
 	}

cmd/erasure-metadata.go

@@ -351,6 +351,17 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	return FileInfo{}, errErasureReadQuorum
 }
 
+func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {
+	diskMTimes := listObjectDiskMtimes(metaArr)
+
+	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta)
+	if diskMaxima >= quorum {
+		return diskMTime
+	}
+
+	return timeSentinel
+}
+
 // pickValidFileInfo - picks one valid FileInfo content and returns from a
 // slice of FileInfo.
 func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (FileInfo, error) {

cmd/erasure-object.go

@@ -179,6 +179,30 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, toObjectErr(err, bucket, object)
 	}
 
+	if !fi.DataShardFixed() {
+		diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks)
+		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+			for index := range onlineDisks {
+				if onlineDisks[index] == OfflineDisk {
+					continue
+				}
+				if !metaArr[index].IsValid() {
+					continue
+				}
+				if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
+					// If disk mTime mismatches it is considered outdated
+					// https://github.com/minio/minio/pull/13803
+					//
+					// This check only is active if we could find maximally
+					// occurring disk mtimes that are somewhat same across
+					// the quorum. Allowing to skip those shards which we
+					// might think are wrong.
+					onlineDisks[index] = OfflineDisk
+				}
+			}
+		}
+	}
+
 	objInfo := fi.ToObjectInfo(bucket, object)
 	if objInfo.DeleteMarker {
 		if opts.VersionID == "" {
@@ -474,7 +498,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, wquorum int, err error) {
 	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
-		return objInfo, getWriteQuorum(len(er.getDisks())), toObjectErr(err, bucket, object)
+		return objInfo, er.defaultWQuorum(), toObjectErr(err, bucket, object)
 	}
 
 	wquorum = fi.Erasure.DataBlocks
@@ -1089,7 +1113,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		// class for objects which have reduced quorum
 		// storage class only needs to be honored for
 		// Read() requests alone which we already do.
-		writeQuorums[i] = getWriteQuorum(len(storageDisks))
+		writeQuorums[i] = er.defaultWQuorum()
 	}
 
 	versionsMap := make(map[string]FileInfoVersions, len(objects))
@@ -1444,14 +1468,16 @@ func (er erasureObjects) addPartial(bucket, object, versionID string, size int64
 }
 
 func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
-	// Lock the object before updating tags.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
+	if !opts.NoLock {
+		// Lock the object before updating metadata.
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx.Cancel)
 	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
 
 	disks := er.getDisks()
 
@@ -1574,7 +1600,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	// Wait for all the routines.
 	mErrs := g.Wait()
 
-	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, getWriteQuorum(len(onlineDisks)))
+	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
 }
 
 // DeleteObjectTags - delete object tags from an existing object

cmd/erasure-server-pool.go

@@ -1684,112 +1684,120 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error
 
-func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
-	errCh := make(chan error)
+func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error, errCh chan<- error) {
 	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	disks, _ := set.getOnlineDisksWithHealing()
+	if len(disks) == 0 {
+		errCh <- errors.New("listAndHeal: No non-healing disks found")
+		return
+	}
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: 1,
+		objQuorum: 1,
+		bucket:    bucket,
+		strict:    false, // Allow less strict matching.
+	}
+
+	path := baseDirFromPrefix(prefix)
+	if path == "" {
+		path = prefix
+	}
+
+	lopts := listPathRawOptions{
+		disks:          disks,
+		bucket:         bucket,
+		path:           path,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       1,
+		reportNotFound: false,
+		agreed: func(entry metaCacheEntry) {
+			if err := healEntry(entry); err != nil {
+				errCh <- err
+				return
+			}
+		},
+		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+			entry, ok := entries.resolve(&resolver)
+			if !ok {
+				// check if we can get one entry atleast
+				// proceed to heal nonetheless.
+				entry, _ = entries.firstFound()
+			}
+
+			if err := healEntry(*entry); err != nil {
+				errCh <- err
+				return
+			}
+		},
+		finished: nil,
+	}
+
+	if err := listPathRaw(ctx, lopts); err != nil {
+		errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
+		return
+	}
+}
+
+func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
+	errCh := make(chan error)
+	healEntry := func(entry metaCacheEntry) error {
+		if entry.isDir() {
+			return nil
+		}
+		// We might land at .metacache, .trash, .multipart
+		// no need to heal them skip, only when bucket
+		// is '.minio.sys'
+		if bucket == minioMetaBucket {
+			if wildcard.Match("buckets/*/.metacache/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("tmp/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("multipart/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("tmp-old/*", entry.name) {
+				return nil
+			}
+		}
+		fivs, err := entry.fileInfoVersions(bucket)
+		if err != nil {
+			return healObjectFn(bucket, entry.name, "")
+		}
+
+		for _, version := range fivs.Versions {
+			if err := healObjectFn(bucket, version.Name, version.VersionID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	go func() {
 		defer close(errCh)
-		defer cancel()
 
+		var wg sync.WaitGroup
 		for _, erasureSet := range z.serverPools {
-			var wg sync.WaitGroup
 			for _, set := range erasureSet.sets {
-				set := set
 				wg.Add(1)
-				go func() {
+				go func(set *erasureObjects) {
 					defer wg.Done()
 
-					disks, _ := set.getOnlineDisksWithHealing()
-					if len(disks) == 0 {
-						cancel()
-						errCh <- errors.New("HealObjects: No non-healing disks found")
-						return
-					}
-
-					healEntry := func(entry metaCacheEntry) {
-						if entry.isDir() {
-							return
-						}
-						// We might land at .metacache, .trash, .multipart
-						// no need to heal them skip, only when bucket
-						// is '.minio.sys'
-						if bucket == minioMetaBucket {
-							if wildcard.Match("buckets/*/.metacache/*", entry.name) {
-								return
-							}
-							if wildcard.Match("tmp/*", entry.name) {
-								return
-							}
-							if wildcard.Match("multipart/*", entry.name) {
-								return
-							}
-							if wildcard.Match("tmp-old/*", entry.name) {
-								return
-							}
-						}
-						fivs, err := entry.fileInfoVersions(bucket)
-						if err != nil {
-							if err := healObject(bucket, entry.name, ""); err != nil {
-								cancel()
-								errCh <- err
-								return
-							}
-							return
-						}
-
-						for _, version := range fivs.Versions {
-							if err := healObject(bucket, version.Name, version.VersionID); err != nil {
-								cancel()
-								errCh <- err
-								return
-							}
-						}
-					}
-
-					// How to resolve partial results.
-					resolver := metadataResolutionParams{
-						dirQuorum: 1,
-						objQuorum: 1,
-						bucket:    bucket,
-						strict:    false, // Allow less strict matching.
-					}
-
-					path := baseDirFromPrefix(prefix)
-					if path == "" {
-						path = prefix
-					}
-
-					lopts := listPathRawOptions{
-						disks:          disks,
-						bucket:         bucket,
-						path:           path,
-						recursive:      true,
-						forwardTo:      "",
-						minDisks:       1,
-						reportNotFound: false,
-						agreed:         healEntry,
-						partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-							entry, ok := entries.resolve(&resolver)
-							if !ok {
-								// check if we can get one entry atleast
-								// proceed to heal nonetheless.
-								entry, _ = entries.firstFound()
-							}
-
-							healEntry(*entry)
-						},
-						finished: nil,
-					}
-
-					if err := listPathRaw(ctx, lopts); err != nil {
-						cancel()
-						errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
-						return
-					}
-				}()
+					listAndHeal(ctx, bucket, prefix, set, healEntry, errCh)
+				}(set)
 			}
-			wg.Wait()
 		}
+		wg.Wait()
 	}()
 	return <-errCh
 }
@@ -1797,14 +1805,33 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	object = encodeDirObject(object)
 
-	for _, pool := range z.serverPools {
-		result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
-		result.Object = decodeDirObject(result.Object)
-		if err != nil {
-			return result, err
-		}
-		return result, nil
+	errs := make([]error, len(z.serverPools))
+	results := make([]madmin.HealResultItem, len(z.serverPools))
+	var wg sync.WaitGroup
+	for idx, pool := range z.serverPools {
+		wg.Add(1)
+		go func(idx int, pool *erasureSets) {
+			defer wg.Done()
+			result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
+			result.Object = decodeDirObject(result.Object)
+			errs[idx] = err
+			results[idx] = result
+		}(idx, pool)
 	}
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return madmin.HealResultItem{}, err
+		}
+	}
+
+	for _, result := range results {
+		if result.Object != "" {
+			return result, nil
+		}
+	}
+
 	if versionID != "" {
 		return madmin.HealResultItem{}, VersionNotFound{
 			Bucket:    bucket,

cmd/erasure.go

@@ -84,6 +84,19 @@ func (er erasureObjects) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+// defaultWQuorum write quorum based on setDriveCount and defaultParityCount
+func (er erasureObjects) defaultWQuorum() int {
+	dataCount := er.setDriveCount - er.defaultParityCount
+	if dataCount == er.defaultParityCount {
+		return dataCount + 1
+	}
+	return dataCount
+}
+
+func (er erasureObjects) defaultRQuorum() int {
+	return er.setDriveCount - er.defaultParityCount
+}
+
 // byDiskTotal is a collection satisfying sort.Interface.
 type byDiskTotal []madmin.Disk
 

cmd/gateway-main.go

@@ -335,11 +335,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// - compression
 	verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
 
-	// Prints the formatted startup message once object layer is initialized.
-	if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
-		// Check update mode.
-		checkUpdate(globalMinioModeGatewayPrefix + gatewayName)
-	}
+	// Check for updates in non-blocking manner.
+	go func() {
+		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
+			// Check for new updates from dl.min.io.
+			checkUpdate(getMinioMode())
+		}
+	}()
 
 	if !globalCLIContext.Quiet {
 		// Print gateway startup message.

cmd/global-heal.go

@@ -168,9 +168,14 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	bgSeq := mustGetHealSequence(ctx)
 	scanMode := globalHealConfig.ScanMode()
 
+	// Make sure to copy since `buckets slice`
+	// is modified in place by tracker.
+	healBuckets := make([]string, len(buckets))
+	copy(healBuckets, buckets)
+
 	var retErr error
 	// Heal all buckets with all objects
-	for _, bucket := range buckets {
+	for _, bucket := range healBuckets {
 		if tracker.isHealed(bucket) {
 			continue
 		}
@@ -318,6 +323,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		default:
 			tracker.bucketDone(bucket)
 			logger.LogIf(ctx, tracker.update(ctx))
+			logger.Info("Healing bucket %s content on %s erasure set complete",
+				bucket, humanize.Ordinal(tracker.SetIndex+1))
 		}
 	}
 	tracker.Object = ""

cmd/iam-dummy-store.go

@@ -22,7 +22,6 @@ import (
 	"sync"
 
 	"github.com/minio/minio/internal/auth"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 type iamDummyStore struct {
@@ -64,7 +63,7 @@ func (ids *iamDummyStore) migrateBackendFormat(context.Context) error {
 	return nil
 }
 
-func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
+func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
 	v, ok := ids.iamPolicyDocsMap[policy]
 	if !ok {
 		return errNoSuchPolicy
@@ -73,7 +72,7 @@ func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m ma
 	return nil
 }
 
-func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
 	for k, v := range ids.iamPolicyDocsMap {
 		m[k] = v
 	}
@@ -154,7 +153,7 @@ func (ids *iamDummyStore) deleteIAMConfig(ctx context.Context, path string) erro
 	return nil
 }
 
-func (ids *iamDummyStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (ids *iamDummyStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return nil
 }
 

cmd/iam-etcd-store.go

@@ -32,7 +32,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/iam/policy"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	etcd "go.etcd.io/etcd/client/v3"
 )
@@ -115,7 +114,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, it
 	return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
 }
 
-func getIAMConfig(item interface{}, data []byte, itemPath string) error {
+func decryptData(data []byte, itemPath string) ([]byte, error) {
 	var err error
 	if !utf8.Valid(data) && GlobalKMS != nil {
 		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
@@ -128,10 +127,18 @@ func getIAMConfig(item interface{}, data []byte, itemPath string) error {
 				minioMetaBucket: itemPath,
 			})
 			if err != nil {
-				return err
+				return nil, err
 			}
 		}
 	}
+	return data, nil
+}
+
+func getIAMConfig(item interface{}, data []byte, itemPath string) error {
+	data, err := decryptData(data, itemPath)
+	if err != nil {
+		return err
+	}
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	return json.Unmarshal(data, item)
 }
@@ -144,6 +151,14 @@ func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item interface{}, pa
 	return getIAMConfig(item, data, path)
 }
 
+func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) {
+	data, err := readKeyEtcd(ctx, ies.client, path)
+	if err != nil {
+		return nil, err
+	}
+	return decryptData(data, path)
+}
+
 func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error {
 	return deleteKeyEtcd(ctx, ies.client, path)
 }
@@ -263,34 +278,46 @@ func (ies *IAMEtcdStore) migrateBackendFormat(ctx context.Context) error {
 	return ies.migrateToV1(ctx)
 }
 
-func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
-	var p iampolicy.Policy
-	err := ies.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
+func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+	data, err := ies.loadIAMConfigBytes(ctx, getPolicyDocPath(policy))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
 		}
 		return err
 	}
+
+	var p PolicyDoc
+	err = p.parseJSON(data)
+	if err != nil {
+		return err
+	}
+
 	m[policy] = p
 	return nil
 }
 
-func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]iampolicy.Policy) error {
-	var p iampolicy.Policy
-	err := getIAMConfig(&p, kvs.Value, string(kvs.Key))
+func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]PolicyDoc) error {
+	data, err := decryptData(kvs.Value, string(kvs.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
 		}
 		return err
 	}
+
+	var p PolicyDoc
+	err = p.parseJSON(data)
+	if err != nil {
+		return err
+	}
+
 	policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key)))
 	m[policy] = p
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
 	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
 	//  Retrieve all keys and values to avoid too many calls to etcd in case of
@@ -473,7 +500,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 }
 
-func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }
 

cmd/iam-object-store.go

@@ -29,7 +29,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // IAMObjectStore implements IAMStorageAPI
@@ -218,19 +217,27 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{}
 	return saveConfig(ctx, iamOS.objAPI, objPath, data)
 }
 
-func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item interface{}, objPath string) error {
-	data, err := readConfig(ctx, iamOS.objAPI, objPath)
+func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) {
+	data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath)
 	if err != nil {
-		return err
+		return nil, meta, err
 	}
 	if !utf8.Valid(data) && GlobalKMS != nil {
 		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, objPath),
 		})
 		if err != nil {
-			return err
+			return nil, meta, err
 		}
 	}
+	return data, meta, nil
+}
+
+func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item interface{}, objPath string) error {
+	data, _, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, objPath)
+	if err != nil {
+		return err
+	}
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	return json.Unmarshal(data, item)
 }
@@ -239,20 +246,34 @@ func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) e
 	return deleteConfig(ctx, iamOS.objAPI, path)
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
-	var p iampolicy.Policy
-	err := iamOS.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
+func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+	data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
 		}
 		return err
 	}
+
+	var p PolicyDoc
+	err = p.parseJSON(data)
+	if err != nil {
+		return err
+	}
+
+	if p.Version == 0 {
+		// This means that policy was in the old version (without any
+		// timestamp info). We fetch the mod time of the file and save
+		// that as create and update date.
+		p.CreateDate = objInfo.ModTime
+		p.UpdateDate = objInfo.ModTime
+	}
+
 	m[policy] = p
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPoliciesPrefix) {
 		if item.Err != nil {
 			return item.Err
@@ -385,7 +406,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 	return nil
 }
 
-func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }
 

cmd/iam-store.go

@@ -24,8 +24,10 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/dustin/go-humanize"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
@@ -169,6 +171,64 @@ func newMappedPolicy(policy string) MappedPolicy {
 	return MappedPolicy{Version: 1, Policies: policy}
 }
 
+// PolicyDoc represents an IAM policy with some metadata.
+type PolicyDoc struct {
+	Version    int `json:",omitempty"`
+	Policy     iampolicy.Policy
+	CreateDate time.Time `json:",omitempty"`
+	UpdateDate time.Time `json:",omitempty"`
+}
+
+func newPolicyDoc(p iampolicy.Policy) PolicyDoc {
+	now := UTCNow().Round(time.Millisecond)
+	return PolicyDoc{
+		Version:    1,
+		Policy:     p,
+		CreateDate: now,
+		UpdateDate: now,
+	}
+}
+
+// defaultPolicyDoc - used to wrap a default policy as PolicyDoc.
+func defaultPolicyDoc(p iampolicy.Policy) PolicyDoc {
+	return PolicyDoc{
+		Version: 1,
+		Policy:  p,
+	}
+}
+
+func (d *PolicyDoc) update(p iampolicy.Policy) {
+	now := UTCNow().Round(time.Millisecond)
+	d.UpdateDate = now
+	if d.CreateDate.IsZero() {
+		d.CreateDate = now
+	}
+	d.Policy = p
+}
+
+// parseJSON parses both the old and the new format for storing policy
+// definitions.
+//
+// The on-disk format of policy definitions has changed (around early 12/2021)
+// from iampolicy.Policy to PolicyDoc. To avoid a migration, loading supports
+// both the old and the new formats.
+func (d *PolicyDoc) parseJSON(data []byte) error {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	var doc PolicyDoc
+	err := json.Unmarshal(data, &doc)
+	if err != nil {
+		err2 := json.Unmarshal(data, &doc.Policy)
+		if err2 != nil {
+			// Just return the first error.
+			return err
+		}
+		d.Policy = doc.Policy
+		return nil
+	}
+	*d = doc
+	return nil
+}
+
 // key options
 type options struct {
 	ttl int64 // expiry in seconds
@@ -182,7 +242,7 @@ type iamWatchEvent struct {
 // iamCache contains in-memory cache of IAM data.
 type iamCache struct {
 	// map of policy names to policy definitions
-	iamPolicyDocsMap map[string]iampolicy.Policy
+	iamPolicyDocsMap map[string]PolicyDoc
 	// map of usernames to credentials
 	iamUsersMap map[string]auth.Credentials
 	// map of group names to group info
@@ -197,7 +257,7 @@ type iamCache struct {
 
 func newIamCache() *iamCache {
 	return &iamCache{
-		iamPolicyDocsMap:        map[string]iampolicy.Policy{},
+		iamPolicyDocsMap:        map[string]PolicyDoc{},
 		iamUsersMap:             map[string]auth.Credentials{},
 		iamGroupsMap:            map[string]GroupInfo{},
 		iamUserGroupMemberships: map[string]set.StringSet{},
@@ -332,8 +392,8 @@ type IAMStorageAPI interface {
 
 	getUsersSysType() UsersSysType
 
-	loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error
-	loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error
+	loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error
+	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
 
 	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error
 	loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error
@@ -348,7 +408,7 @@ type IAMStorageAPI interface {
 	loadIAMConfig(ctx context.Context, item interface{}, path string) error
 	deleteIAMConfig(ctx context.Context, path string) error
 
-	savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error
+	savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error
 	saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error
 	saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error
 	saveGroupInfo(ctx context.Context, group string, gi GroupInfo) error
@@ -366,10 +426,10 @@ type iamStorageWatcher interface {
 }
 
 // Set default canned policies only if not already overridden by users.
-func setDefaultCannedPolicies(policies map[string]iampolicy.Policy) {
+func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
 	for _, v := range iampolicy.DefaultPolicies {
 		if _, ok := policies[v.Name]; !ok {
-			policies[v.Name] = v.Definition
+			policies[v.Name] = defaultPolicyDoc(v.Definition)
 		}
 	}
 }
@@ -947,13 +1007,31 @@ func (store *IAMStoreSys) GetPolicy(name string) (iampolicy.Policy, error) {
 		}
 		v, ok := cache.iamPolicyDocsMap[policy]
 		if !ok {
-			return v, errNoSuchPolicy
+			return v.Policy, errNoSuchPolicy
 		}
-		combinedPolicy = combinedPolicy.Merge(v)
+		combinedPolicy = combinedPolicy.Merge(v.Policy)
 	}
 	return combinedPolicy, nil
 }
 
+// GetPolicyDoc - gets the policy doc which has the policy and some metadata.
+// Exactly one policy must be specified here.
+func (store *IAMStoreSys) GetPolicyDoc(name string) (r PolicyDoc, err error) {
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return r, errInvalidArgument
+	}
+
+	cache := store.rlock()
+	defer store.runlock()
+
+	v, ok := cache.iamPolicyDocsMap[name]
+	if !ok {
+		return r, errNoSuchPolicy
+	}
+	return v, nil
+}
+
 // SetPolicy - creates a policy with name.
 func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) error {
 
@@ -964,11 +1042,21 @@ func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iam
 	cache := store.lock()
 	defer store.unlock()
 
-	if err := store.savePolicyDoc(ctx, name, policy); err != nil {
+	var (
+		d  PolicyDoc
+		ok bool
+	)
+	if d, ok = cache.iamPolicyDocsMap[name]; ok {
+		d.update(policy)
+	} else {
+		d = newPolicyDoc(policy)
+	}
+
+	if err := store.savePolicyDoc(ctx, name, d); err != nil {
 		return err
 	}
 
-	cache.iamPolicyDocsMap[name] = policy
+	cache.iamPolicyDocsMap[name] = d
 	return nil
 
 }
@@ -979,7 +1067,7 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 	cache := store.lock()
 	defer store.unlock()
 
-	m := map[string]iampolicy.Policy{}
+	m := map[string]PolicyDoc{}
 	err := store.loadPolicyDocs(ctx, m)
 	if err != nil {
 		return nil, err
@@ -992,8 +1080,8 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 
 	ret := map[string]iampolicy.Policy{}
 	for k, v := range m {
-		if bucketName == "" || v.MatchResource(bucketName) {
-			ret[k] = v
+		if bucketName == "" || v.Policy.MatchResource(bucketName) {
+			ret[k] = v.Policy
 		}
 	}
 
@@ -1011,9 +1099,9 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 		}
 		p, found := cache.iamPolicyDocsMap[policy]
 		if found {
-			if bucketName == "" || p.MatchResource(bucketName) {
+			if bucketName == "" || p.Policy.MatchResource(bucketName) {
 				policies = append(policies, policy)
-				combinedPolicy = combinedPolicy.Merge(p)
+				combinedPolicy = combinedPolicy.Merge(p.Policy)
 			}
 		}
 	}
@@ -1304,9 +1392,11 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	return err
 }
 
-// SetTempUser - saves temporary credential to storage and cache.
+// SetTempUser - saves temporary (STS) credential to storage and cache. If a
+// policy name is given, it is associated with the parent user specified in the
+// credential.
 func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
-	if accessKey == "" || !cred.IsTemp() || cred.IsExpired() {
+	if accessKey == "" || !cred.IsTemp() || cred.IsExpired() || cred.ParentUser == "" {
 		return errInvalidArgument
 	}
 
@@ -1323,26 +1413,12 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 			return fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy)
 		}
 
-		err := store.saveMappedPolicy(ctx, accessKey, stsUser, false, mp, options{ttl: ttl})
+		err := store.saveMappedPolicy(ctx, cred.ParentUser, stsUser, false, mp, options{ttl: ttl})
 		if err != nil {
 			return err
 		}
 
-		cache.iamUserPolicyMap[accessKey] = mp
-
-		// We are on purpose not persisting the policy map for parent
-		// user, although this is a hack, it is a good enough hack
-		// at this point in time - we need to overhaul our OIDC
-		// usage with service accounts with a more cleaner implementation
-		//
-		// This mapping is necessary to ensure that valid credentials
-		// have necessary ParentUser present - this is mainly for only
-		// webIdentity based STS tokens.
-		if cred.ParentUser != "" && cred.ParentUser != globalActiveCred.AccessKey {
-			if _, ok := cache.iamUserPolicyMap[cred.ParentUser]; !ok {
-				cache.iamUserPolicyMap[cred.ParentUser] = mp
-			}
-		}
+		cache.iamUserPolicyMap[cred.ParentUser] = mp
 	}
 
 	u := newUserIdentity(cred)
@@ -1553,7 +1629,7 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 }
 
 // AddUser - adds/updates long term user account to storage.
-func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, uinfo madmin.UserInfo) error {
+func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
 	cache := store.lock()
 	defer store.unlock()
 
@@ -1566,9 +1642,9 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, uinfo m
 
 	u := newUserIdentity(auth.Credentials{
 		AccessKey: accessKey,
-		SecretKey: uinfo.SecretKey,
+		SecretKey: ureq.SecretKey,
 		Status: func() string {
-			if uinfo.Status == madmin.AccountEnabled {
+			if ureq.Status == madmin.AccountEnabled {
 				return auth.AccountOn
 			}
 			return auth.AccountOff
@@ -1581,25 +1657,7 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, uinfo m
 
 	cache.iamUsersMap[accessKey] = u.Credentials
 
-	// Set policy if specified.
-	if uinfo.PolicyName != "" {
-		policy := uinfo.PolicyName
-		// Handle policy mapping set/update
-		mp := newMappedPolicy(policy)
-		for _, p := range mp.toSlice() {
-			if _, found := cache.iamPolicyDocsMap[policy]; !found {
-				logger.LogIf(GlobalContext, fmt.Errorf("%w: (%s)", errNoSuchPolicy, p))
-				return errNoSuchPolicy
-			}
-		}
-
-		if err := store.saveMappedPolicy(ctx, accessKey, regUser, false, mp); err != nil {
-			return err
-		}
-		cache.iamUserPolicyMap[accessKey] = mp
-	}
 	return nil
-
 }
 
 // UpdateUserSecretKey - sets user secret key to storage.

cmd/iam.go

@@ -477,13 +477,28 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 	return nil
 }
 
-// InfoPolicy - expands the canned policy into its JSON structure.
-func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
+// InfoPolicy - returns the policy definition with some metadata.
+func (sys *IAMSys) InfoPolicy(policyName string) (*madmin.PolicyInfo, error) {
 	if !sys.Initialized() {
-		return iampolicy.Policy{}, errServerNotInitialized
+		return nil, errServerNotInitialized
 	}
 
-	return sys.store.GetPolicy(policyName)
+	d, err := sys.store.GetPolicyDoc(policyName)
+	if err != nil {
+		return nil, err
+	}
+
+	pdata, err := json.Marshal(d.Policy)
+	if err != nil {
+		return nil, err
+	}
+
+	return &madmin.PolicyInfo{
+		PolicyName: policyName,
+		Policy:     pdata,
+		CreateDate: d.CreateDate,
+		UpdateDate: d.UpdateDate,
+	}, nil
 }
 
 // ListPolicies - lists all canned policies.
@@ -521,12 +536,26 @@ func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy
 }
 
 // DeleteUser - delete user (only for long-term users not STS users).
-func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string) error {
+func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
-	return sys.store.DeleteUser(ctx, accessKey, regUser)
+	if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil {
+		return err
+	}
+
+	// Notify all other MinIO peers to delete user.
+	if notifyPeers && !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.DeleteUser(accessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	return nil
 }
 
 // CurrentPolicies - returns comma separated policy string, from
@@ -553,7 +582,45 @@ func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp b
 	}
 }
 
-// SetTempUser - set temporary user credentials, these credentials have an expiry.
+// SetTempUser - set temporary user credentials, these credentials have an
+// expiry. The permissions for these STS credentials is determined in one of the
+// following ways:
+//
+// - RoleARN - if a role-arn is specified in the request, the STS credential's
+// policy is the role's policy.
+//
+// - inherited from parent - this is the case for AssumeRole API, where the
+// parent user is an actual real user with their own (permanent) credentials and
+// policy association.
+//
+// - inherited from "virtual" parent - this is the case for AssumeRoleWithLDAP
+// where the parent user is the DN of the actual LDAP user. The parent user
+// itself cannot login, but the policy associated with them determines the base
+// policy for the STS credential. The policy mapping can be updated by the
+// administrator.
+//
+// - from `Subject.CommonName` field from the STS request for
+// AssumeRoleWithCertificate. In this case, the policy for the STS credential
+// has the same name as the value of this field.
+//
+// - from special JWT claim from STS request for AssumeRoleWithOIDC API (when
+// not using RoleARN). The claim value can be a string or a list and refers to
+// the names of access policies.
+//
+// For all except the RoleARN case, the implementation is the same - the policy
+// for the STS credential is associated with a parent user. For the
+// AssumeRoleWithCertificate case, the "virtual" parent user is the value of the
+// `Subject.CommonName` field. For the OIDC (without RoleARN) case the "virtual"
+// parent is derived as a concatenation of the `sub` and `iss` fields. The
+// policies applicable to the STS credential are associated with this "virtual"
+// parent.
+//
+// When a policyName is given to this function, the policy association is
+// created and stored in the IAM store. Thus, it should NOT be given for the
+// role-arn case (because the role-to-policy mapping is separately stored
+// elsewhere), the AssumeRole case (because the parent user is real and their
+// policy is associated via policy-set API) and the AssumeRoleWithLDAP case
+// (because the policy association is made via policy-set API).
 func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
@@ -859,7 +926,7 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 }
 
 // DeleteServiceAccount - delete a service account
-func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) error {
+func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, notifyPeers bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
@@ -869,12 +936,25 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 		return nil
 	}
 
-	return sys.store.DeleteUser(ctx, accessKey, svcUser)
+	if err := sys.store.DeleteUser(ctx, accessKey, svcUser); err != nil {
+		return err
+	}
+
+	if notifyPeers && !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.DeleteServiceAccount(accessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	return nil
 }
 
 // CreateUser - create new user credentials and policy, if user already exists
 // they shall be rewritten with new inputs.
-func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, uinfo madmin.UserInfo) error {
+func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
@@ -887,11 +967,11 @@ func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, uinfo madmi
 		return auth.ErrInvalidAccessKeyLength
 	}
 
-	if !auth.IsSecretKeyValid(uinfo.SecretKey) {
+	if !auth.IsSecretKeyValid(ureq.SecretKey) {
 		return auth.ErrInvalidSecretKeyLength
 	}
 
-	err := sys.store.AddUser(ctx, accessKey, uinfo)
+	err := sys.store.AddUser(ctx, accessKey, ureq)
 	if err != nil {
 		return err
 	}
@@ -1176,7 +1256,7 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 
 	err := sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
 	if err != nil {
-		return nil
+		return err
 	}
 
 	// Notify all other MinIO peers to reload policy
@@ -1373,34 +1453,28 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 		}
 		policies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
 	} else {
-		// If roleArn is not used, we fall back to using policy claim
-		// from JWT.
-		policySet, ok := args.GetPolicies(iamPolicyClaimNameOpenID())
-		if !ok {
-			// When claims are set, it should have a policy claim field.
-			return false
-		}
-		// When claims are set, it should have policies as claim.
-		if policySet.IsEmpty() {
-			// No policy, no access!
+		// Lookup the parent user's mapping if there's no role-ARN.
+		var err error
+		policies, err = sys.store.PolicyDBGet(parentUser, false, args.Groups...)
+		if err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err))
 			return false
 		}
+		if len(policies) == 0 {
+			// TODO (deprecated in Dec 2021): Only need to handle
+			// behavior for STS credentials created in older
+			// releases. Otherwise, reject such cases, once older
+			// behavior is deprecated.
 
-		// If policy is available for given user, check the policy.
-		mp, ok := sys.store.GetMappedPolicy(args.AccountName, false)
-		if !ok {
-			// No policy set for the user that we can find, no access!
-			return false
+			// If there is no parent policy mapping, we fall back to
+			// using policy claim from JWT.
+			policySet, ok := args.GetPolicies(iamPolicyClaimNameOpenID())
+			if !ok {
+				// When claims are set, it should have a policy claim field.
+				return false
+			}
+			policies = policySet.ToSlice()
 		}
-
-		if !policySet.Equals(mp.policySet()) {
-			// When claims has a policy, it should match the
-			// policy of args.AccountName which server remembers.
-			// if not reject such requests.
-			return false
-		}
-
-		policies = policySet.ToSlice()
 	}
 
 	combinedPolicy, err := sys.store.GetPolicy(strings.Join(policies, ","))

cmd/last-minute.go

@@ -102,39 +102,21 @@ type LastMinuteLatencies struct {
 	LastSec int64
 }
 
-// Clone safely returns a copy for a LastMinuteLatencies structure
-func (l *LastMinuteLatencies) Clone() LastMinuteLatencies {
-	n := LastMinuteLatencies{}
-	n.LastSec = l.LastSec
-	for i := range l.Totals {
-		for j := range l.Totals[i] {
-			n.Totals[i][j] = AccElem{
-				Total: l.Totals[i][j].Total,
-				N:     l.Totals[i][j].N,
-			}
-		}
-	}
-	return n
-}
-
 // Merge safely merges two LastMinuteLatencies structures into one
 func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLatencies) {
-	cl := l.Clone()
-	co := o.Clone()
-
-	if cl.LastSec > co.LastSec {
-		co.forwardTo(cl.LastSec)
-		merged.LastSec = cl.LastSec
+	if l.LastSec > o.LastSec {
+		o.forwardTo(l.LastSec)
+		merged.LastSec = l.LastSec
 	} else {
-		cl.forwardTo(co.LastSec)
-		merged.LastSec = co.LastSec
+		l.forwardTo(o.LastSec)
+		merged.LastSec = o.LastSec
 	}
 
-	for i := range cl.Totals {
-		for j := range cl.Totals[i] {
+	for i := range merged.Totals {
+		for j := range merged.Totals[i] {
 			merged.Totals[i][j] = AccElem{
-				Total: cl.Totals[i][j].Total + co.Totals[i][j].Total,
-				N:     cl.Totals[i][j].N + co.Totals[i][j].N,
+				Total: l.Totals[i][j].Total + o.Totals[i][j].Total,
+				N:     l.Totals[i][j].N + o.Totals[i][j].N,
 			}
 		}
 	}

cmd/metacache-server-pool.go

@@ -199,13 +199,15 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 		entries.truncate(0)
 		go func() {
 			rpc := globalNotificationSys.restClientFromHash(o.Bucket)
-			ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-			defer cancel()
-			c, err := rpc.GetMetacacheListing(ctx, *o)
-			if err == nil {
-				c.error = "no longer used"
-				c.status = scanStateError
-				rpc.UpdateMetacacheListing(ctx, *c)
+			if rpc != nil {
+				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
+				defer cancel()
+				c, err := rpc.GetMetacacheListing(ctx, *o)
+				if err == nil {
+					c.error = "no longer used"
+					c.status = scanStateError
+					rpc.UpdateMetacacheListing(ctx, *c)
+				}
 			}
 		}()
 		o.ID = ""

cmd/metacache-set.go

@@ -544,7 +544,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 	// Special case: ask all disks if the drive count is 4
 	if askDisks == -1 || er.setDriveCount == 4 {
 		askDisks = len(disks) // with 'strict' quorum list on all online disks.
-		listingQuorum = getReadQuorum(er.setDriveCount)
+		listingQuorum = er.defaultRQuorum()
 	}
 	if askDisks == 0 {
 		askDisks = globalAPIConfig.getListQuorum()

cmd/peer-rest-client.go

@@ -720,6 +720,11 @@ func (client *peerRESTClient) GetLocalDiskIDs(ctx context.Context) (diskIDs []st
 
 // GetMetacacheListing - get a new or existing metacache.
 func (client *peerRESTClient) GetMetacacheListing(ctx context.Context, o listPathOptions) (*metacache, error) {
+	if client == nil {
+		resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(o)
+		return &resp, nil
+	}
+
 	var reader bytes.Buffer
 	err := gob.NewEncoder(&reader).Encode(o)
 	if err != nil {
@@ -737,6 +742,9 @@ func (client *peerRESTClient) GetMetacacheListing(ctx context.Context, o listPat
 
 // UpdateMetacacheListing - update an existing metacache it will unconditionally be updated to the new state.
 func (client *peerRESTClient) UpdateMetacacheListing(ctx context.Context, m metacache) (metacache, error) {
+	if client == nil {
+		return localMetacacheMgr.updateCacheEntry(m)
+	}
 	b, err := m.MarshalMsg(nil)
 	if err != nil {
 		return m, err

cmd/peer-rest-server.go

@@ -157,7 +157,7 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err := globalIAMSys.DeleteServiceAccount(r.Context(), accessKey); err != nil {
+	if err := globalIAMSys.DeleteServiceAccount(r.Context(), accessKey, false); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -209,7 +209,7 @@ func (s *peerRESTServer) DeleteUserHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err := globalIAMSys.DeleteUser(r.Context(), accessKey); err != nil {
+	if err := globalIAMSys.DeleteUser(r.Context(), accessKey, false); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -1113,7 +1113,7 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 
 	enc := gob.NewEncoder(w)
 
-	ch := ReportMetrics(r.Context(), GetGeneratorsForPeer)
+	ch := ReportMetrics(r.Context(), peerMetricsGroups)
 	for m := range ch {
 		if err := enc.Encode(m); err != nil {
 			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))

cmd/server-main.go

@@ -460,10 +460,13 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
-	if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
-		// Check for new updates from dl.min.io.
-		checkUpdate(getMinioMode())
-	}
+	// Check for updates in non-blocking manner.
+	go func() {
+		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
+			// Check for new updates from dl.min.io.
+			checkUpdate(getMinioMode())
+		}
+	}()
 
 	if !globalActiveCred.IsValid() && globalIsDistErasure {
 		globalActiveCred = auth.DefaultCredentials
@@ -588,8 +591,9 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
+	// initialize the new disk cache objects.
 	if globalCacheConfig.Enabled {
-		// initialize the new disk cache objects.
+		logStartupMessage(color.Yellow("WARNING: Disk caching is deprecated for single/multi drive MinIO setups. Please migrate to using MinIO S3 gateway instead of disk caching"))
 		var cacheAPI CacheObjectLayer
 		cacheAPI, err = newServerCacheObjects(GlobalContext, globalCacheConfig)
 		logger.FatalIf(err, "Unable to initialize disk caching")

cmd/signature-v4-utils.go

@@ -154,6 +154,11 @@ func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, A
 		// Check if the access key is part of users credentials.
 		ucred, ok := globalIAMSys.GetUser(r.Context(), accessKey)
 		if !ok {
+			// Credentials will be invalid but and disabled
+			// return a different error in such a scenario.
+			if ucred.Status == auth.AccountOff {
+				return cred, false, ErrAccessKeyDisabled
+			}
 			return cred, false, ErrInvalidAccessKeyID
 		}
 		cred = ucred

cmd/signature-v4-utils_test.go

@@ -76,7 +76,7 @@ func TestCheckValid(t *testing.T) {
 		t.Fatalf("unable create credential, %s", err)
 	}
 
-	globalIAMSys.CreateUser(ctx, ucreds.AccessKey, madmin.UserInfo{
+	globalIAMSys.CreateUser(ctx, ucreds.AccessKey, madmin.AddOrUpdateUserReq{
 		SecretKey: ucreds.SecretKey,
 		Status:    madmin.AccountEnabled,
 	})

cmd/site-replication.go

@@ -40,7 +40,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	sreplication "github.com/minio/minio/internal/bucket/replication"
-	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -56,11 +55,26 @@ const (
 )
 
 var (
-	errSRCannotJoin     = errors.New("this site is already configured for site-replication")
-	errSRDuplicateSites = errors.New("duplicate sites provided for site-replication")
-	errSRSelfNotFound   = errors.New("none of the given sites correspond to the current one")
-	errSRPeerNotFound   = errors.New("peer not found")
-	errSRNotEnabled     = errors.New("site replication is not enabled")
+	errSRCannotJoin = SRError{
+		Cause: errors.New("this site is already configured for site-replication"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRDuplicateSites = SRError{
+		Cause: errors.New("duplicate sites provided for site-replication"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRSelfNotFound = SRError{
+		Cause: errors.New("none of the given sites correspond to the current one"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRPeerNotFound = SRError{
+		Cause: errors.New("peer not found"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRNotEnabled = SRError{
+		Cause: errors.New("site replication is not enabled"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
 )
 
 func errSRInvalidRequest(err error) SRError {
@@ -309,7 +323,7 @@ func (c *SiteReplicationSys) getSiteStatuses(ctx context.Context, sites []madmin
 }
 
 // AddPeerClusters - add cluster sites for replication configuration.
-func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmin.PeerSite) (madmin.ReplicateAddStatus, SRError) {
+func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmin.PeerSite) (madmin.ReplicateAddStatus, error) {
 	sites, serr := c.getSiteStatuses(ctx, psites)
 	if serr.Cause != nil {
 		return madmin.ReplicateAddStatus{}, serr
@@ -335,7 +349,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 	for i, v := range sites {
 		// deploymentIDs must be unique
 		if deploymentIDsSet.Contains(v.DeploymentID) {
-			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errSRDuplicateSites)
+			return madmin.ReplicateAddStatus{}, errSRDuplicateSites
 		}
 		deploymentIDsSet.Add(v.DeploymentID)
 
@@ -351,32 +365,32 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 	if c.enabled {
 		// If current cluster is already SR enabled and no new site being added ,fail.
 		if currDeploymentIDsSet.Equals(deploymentIDsSet) {
-			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errSRCannotJoin)
+			return madmin.ReplicateAddStatus{}, errSRCannotJoin
 		}
 		if len(currDeploymentIDsSet.Intersection(deploymentIDsSet)) != len(currDeploymentIDsSet) {
 			diffSlc := getMissingSiteNames(currDeploymentIDsSet, deploymentIDsSet, currSites.Sites)
-			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("All existing replicated sites must be specified - missing %s", strings.Join(diffSlc, " ")))
+			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("all existing replicated sites must be specified - missing %s", strings.Join(diffSlc, " ")))
 		}
 	}
 	// For this `add` API, either all clusters must be empty or the local
 	// cluster must be the only one having some buckets.
 
 	if localHasBuckets && nonLocalPeerWithBuckets != "" {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("Only one cluster may have data when configuring site replication"))
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("only one cluster may have data when configuring site replication"))
 	}
 
 	if !localHasBuckets && nonLocalPeerWithBuckets != "" {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("Please send your request to the cluster containing data/buckets: %s", nonLocalPeerWithBuckets))
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("please send your request to the cluster containing data/buckets: %s", nonLocalPeerWithBuckets))
 	}
 
 	// validate that all clusters are using the same (LDAP based)
 	// external IDP.
-	pass, verr := c.validateIDPSettings(ctx, sites)
-	if verr.Cause != nil {
-		return madmin.ReplicateAddStatus{}, verr
+	pass, err := c.validateIDPSettings(ctx, sites)
+	if err != nil {
+		return madmin.ReplicateAddStatus{}, err
 	}
 	if !pass {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("All cluster sites must have the same (LDAP) IDP settings."))
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("all cluster sites must have the same (LDAP) IDP settings"))
 	}
 
 	// FIXME: Ideally, we also need to check if there are any global IAM
@@ -413,7 +427,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 		return madmin.ReplicateAddStatus{}, errSRBackendIssue(err)
 	}
 
-	joinReq := madmin.SRInternalJoinReq{
+	joinReq := madmin.SRPeerJoinReq{
 		SvcAcctAccessKey: svcCred.AccessKey,
 		SvcAcctSecretKey: secretKey,
 		Peers:            make(map[string]madmin.PeerInfo),
@@ -429,7 +443,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 
 	addedCount := 0
 	var (
-		peerAddErr SRError
+		peerAddErr error
 		admClient  *madmin.AdminClient
 	)
 
@@ -448,7 +462,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 			break
 		}
 		joinReq.SvcAcctParent = v.AccessKey
-		err = admClient.SRInternalJoin(ctx, joinReq)
+		err = admClient.SRPeerJoin(ctx, joinReq)
 		if err != nil {
 			peerAddErr = errSRPeerResp(fmt.Errorf("unable to link with peer %s: %w", v.Name, err))
 			break
@@ -456,7 +470,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 		addedCount++
 	}
 
-	if peerAddErr.Cause != nil {
+	if peerAddErr != nil {
 		if addedCount == 0 {
 			return madmin.ReplicateAddStatus{}, peerAddErr
 		}
@@ -469,8 +483,9 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 			ErrDetail: peerAddErr.Error(),
 		}
 
-		return partial, SRError{}
+		return partial, nil
 	}
+
 	// Other than handling existing buckets, we can now save the cluster
 	// replication configuration state.
 	state := srState{
@@ -478,29 +493,29 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 		Peers:                   joinReq.Peers,
 		ServiceAccountAccessKey: svcCred.AccessKey,
 	}
-	err = c.saveToDisk(ctx, state)
-	if err != nil {
+
+	if err = c.saveToDisk(ctx, state); err != nil {
 		return madmin.ReplicateAddStatus{
 			Status:    madmin.ReplicateAddStatusPartial,
 			ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err),
-		}, SRError{}
+		}, nil
 	}
 
 	result := madmin.ReplicateAddStatus{
 		Success: true,
 		Status:  madmin.ReplicateAddStatusSuccess,
 	}
-	initialSyncErr := c.syncLocalToPeers(ctx)
-	if initialSyncErr.Code != ErrNone {
-		result.InitialSyncErrorMessage = initialSyncErr.Error()
+
+	if err := c.syncLocalToPeers(ctx); err != nil {
+		result.InitialSyncErrorMessage = err.Error()
 	}
 
-	return result, SRError{}
+	return result, nil
 }
 
 // InternalJoinReq - internal API handler to respond to a peer cluster's request
 // to join.
-func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRInternalJoinReq) SRError {
+func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRPeerJoinReq) error {
 	var ourName string
 	for d, p := range arg.Peers {
 		if d == globalDeploymentID {
@@ -509,7 +524,7 @@ func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRI
 		}
 	}
 	if ourName == "" {
-		return errSRInvalidRequest(errSRSelfNotFound)
+		return errSRSelfNotFound
 	}
 
 	_, _, err := globalIAMSys.GetServiceAccount(ctx, arg.SvcAcctAccessKey)
@@ -528,11 +543,10 @@ func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRI
 		Peers:                   arg.Peers,
 		ServiceAccountAccessKey: arg.SvcAcctAccessKey,
 	}
-	err = c.saveToDisk(ctx, state)
-	if err != nil {
+	if err = c.saveToDisk(ctx, state); err != nil {
 		return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to disk on %s: %v", ourName, err))
 	}
-	return SRError{}
+	return nil
 }
 
 // GetIDPSettings returns info about the configured identity provider. It is
@@ -547,7 +561,7 @@ func (c *SiteReplicationSys) GetIDPSettings(ctx context.Context) madmin.IDPSetti
 	}
 }
 
-func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []PeerSiteInfo) (bool, SRError) {
+func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []PeerSiteInfo) (bool, error) {
 	s := make([]madmin.IDPSettings, 0, len(peers))
 	for _, v := range peers {
 		if v.self {
@@ -560,7 +574,7 @@ func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []Pe
 			return false, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
 		}
 
-		is, err := admClient.SRInternalGetIDPSettings(ctx)
+		is, err := admClient.SRPeerGetIDPSettings(ctx)
 		if err != nil {
 			return false, errSRPeerResp(fmt.Errorf("unable to fetch IDP settings from %s: %v", v.Name, err))
 		}
@@ -569,15 +583,15 @@ func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []Pe
 
 	for _, v := range s {
 		if !v.IsLDAPEnabled {
-			return false, SRError{}
+			return false, nil
 		}
 	}
 	for i := 1; i < len(s); i++ {
 		if s[i] != s[0] {
-			return false, SRError{}
+			return false, nil
 		}
 	}
-	return true, SRError{}
+	return true, nil
 }
 
 // GetClusterInfo - returns site replication information.
@@ -620,7 +634,11 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 		optsMap["location"] = opts.Location
 	}
 	if opts.LockEnabled {
-		optsMap["lockEnabled"] = ""
+		optsMap["lockEnabled"] = "true"
+		optsMap["versioningEnabled"] = "true"
+	}
+	if opts.VersioningEnabled {
+		optsMap["versioningEnabled"] = "true"
 	}
 
 	// Create bucket and enable versioning on all peers.
@@ -636,7 +654,7 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 				return err
 			}
 
-			err = admClient.SRInternalBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)
+			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)
 			logger.LogIf(ctx, c.annotatePeerErr(p.Name, "MakeWithVersioning", err))
 			return err
 		},
@@ -664,7 +682,7 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 				return err
 			}
 
-			err = admClient.SRInternalBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)
+			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)
 			logger.LogIf(ctx, c.annotatePeerErr(p.Name, "ConfigureRepl", err))
 			return err
 		},
@@ -701,7 +719,7 @@ func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRInternalBucketOps(ctx, bucket, op, nil)
+		err = admClient.SRPeerBucketOps(ctx, bucket, op, nil)
 		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "DeleteBucket", err))
 		return err
 	})
@@ -714,6 +732,7 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
+
 	err := objAPI.MakeBucketWithLocation(ctx, bucket, opts)
 	if err != nil {
 		// Check if this is a bucket exists error.
@@ -729,27 +748,25 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 		globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 	}
 
-	// Enable versioning on the bucket.
-	config, err := globalBucketVersioningSys.Get(bucket)
+	meta, err := globalBucketMetadataSys.Get(bucket)
 	if err != nil {
+		logger.LogIf(ctx, c.annotateErr("MakeBucketErr on peer call", err))
 		return wrapSRErr(err)
 	}
-	if !config.Enabled() {
-		verConf := versioning.Versioning{
-			Status: versioning.Enabled,
-		}
-		// FIXME: need to confirm if skipping object lock and
-		// versioning-suspended state checks are valid here.
-		cfgData, err := xml.Marshal(verConf)
-		if err != nil {
-			return wrapSRErr(err)
-		}
-		err = globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, cfgData)
-		if err != nil {
-			logger.LogIf(ctx, c.annotateErr("Versioning enabling error on peer call", err))
-			return wrapSRErr(err)
-		}
+
+	meta.VersioningConfigXML = enabledBucketVersioningConfig
+	if opts.LockEnabled {
+		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
 	}
+
+	if err := meta.Save(context.Background(), objAPI); err != nil {
+		return wrapSRErr(err)
+	}
+
+	globalBucketMetadataSys.Set(bucket, meta)
+
+	// Load updated bucket metadata into memory as new metadata updated.
+	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 	return nil
 }
 
@@ -931,14 +948,24 @@ func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket
 		return errSRNotEnabled
 	}
 
-	// FIXME: need to handle cases where globalDNSConfig is set.
-
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			return err
+		}
+	}
+
 	err := objAPI.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete})
 	if err != nil {
+		if globalDNSConfig != nil {
+			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
+			}
+		}
 		return err
 	}
 
@@ -983,8 +1010,8 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRInternalReplicateIAMItem(ctx, item)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRInternalReplicateIAMItem", err))
+		err = admClient.SRPeerReplicateIAMItem(ctx, item)
+		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRPeerReplicateIAMItem", err))
 		return err
 	})
 	return cErr.summaryErr
@@ -1008,7 +1035,7 @@ func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyNam
 // PeerSvcAccChangeHandler - copies service-account change to local.
 func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange) error {
 	if change == nil {
-		return errInvalidArgument
+		return errSRInvalidRequest(errInvalidArgument)
 	}
 	switch {
 	case change.Create != nil:
@@ -1053,18 +1080,11 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 		}
 
 	case change.Delete != nil:
-		err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey)
+		err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		for _, nerr := range globalNotificationSys.DeleteServiceAccount(change.Delete.AccessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-
 	}
 
 	return nil
@@ -1073,7 +1093,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 // PeerPolicyMappingHandler - copies policy mapping to local.
 func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping) error {
 	if mapping == nil {
-		return errInvalidArgument
+		return errSRInvalidRequest(errInvalidArgument)
 	}
 	err := globalIAMSys.PolicyDBSet(ctx, mapping.UserOrGroup, mapping.Policy, mapping.IsGroup)
 	if err != nil {
@@ -1085,7 +1105,7 @@ func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mappi
 // PeerSTSAccHandler - replicates STS credential locally.
 func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential) error {
 	if stsCred == nil {
-		return errInvalidArgument
+		return errSRInvalidRequest(errInvalidArgument)
 	}
 
 	// Verify the session token of the stsCred
@@ -1149,8 +1169,8 @@ func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRB
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRInternalReplicateBucketMeta(ctx, item)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRInternalReplicateBucketMeta", err))
+		err = admClient.SRPeerReplicateBucketMeta(ctx, item)
+		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRPeerReplicateBucketMeta", err))
 		return err
 	})
 	return cErr.summaryErr
@@ -1261,7 +1281,7 @@ func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID st
 func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
 	creds, ok := globalIAMSys.store.GetUser(c.state.ServiceAccountAccessKey)
 	if !ok {
-		return nil, errors.New("site replication service account not found!")
+		return nil, errors.New("site replication service account not found")
 	}
 	return &creds, nil
 }
@@ -1269,7 +1289,7 @@ func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
 // syncLocalToPeers is used when initially configuring site replication, to
 // copy existing buckets, their settings, service accounts and policies to all
 // new peers.
-func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) SRError {
+func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
 	// If local has buckets, enable versioning on them, create them on peers
 	// and setup replication rules.
 	objAPI := newObjectLayerFn()
@@ -1518,7 +1538,7 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) SRError {
 		}
 	}
 
-	return SRError{}
+	return nil
 }
 
 // Concurrency helpers
@@ -1608,6 +1628,13 @@ func (c *SiteReplicationSys) annotatePeerErr(dstPeer string, annotation string,
 	return fmt.Errorf("%s->%s: %s: %v", c.state.Name, dstPeer, annotation, err)
 }
 
+// isEnabled returns true if site replication is enabled
+func (c *SiteReplicationSys) isEnabled() bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.enabled
+}
+
 // Other helpers
 
 // newRemoteClusterHTTPTransport returns a new http configuration

cmd/storage-datatypes.go

@@ -184,6 +184,25 @@ type FileInfo struct {
 	// no other caller must set this value other than multi-object delete call.
 	// usage in other calls in undefined please avoid.
 	Idx int `msg:"i"`
+
+	// DiskMTime indicates the mtime of the xl.meta on disk
+	// This is mainly used for detecting a particular issue
+	// reported in https://github.com/minio/minio/pull/13803
+	DiskMTime time.Time `msg:"dmt"`
+}
+
+// Equals checks if fi(FileInfo) matches ofi(FileInfo)
+func (fi FileInfo) Equals(ofi FileInfo) (ok bool) {
+	if !fi.MetadataEquals(ofi) {
+		return false
+	}
+	if !fi.ReplicationInfoEquals(ofi) {
+		return false
+	}
+	if !fi.TransitionInfoEquals(ofi) {
+		return false
+	}
+	return fi.ModTime.Equal(ofi.ModTime)
 }
 
 // GetDataDir returns an expected dataDir given FileInfo

cmd/storage-datatypes_gen.go

@@ -550,8 +550,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 25 {
-		err = msgp.ArrayError{Wanted: 25, Got: zb0001}
+	if zb0001 != 26 {
+		err = msgp.ArrayError{Wanted: 26, Got: zb0001}
 		return
 	}
 	z.Volume, err = dc.ReadString()
@@ -716,13 +716,18 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "Idx")
 		return
 	}
+	z.DiskMTime, err = dc.ReadTime()
+	if err != nil {
+		err = msgp.WrapError(err, "DiskMTime")
+		return
+	}
 	return
 }
 
 // EncodeMsg implements msgp.Encodable
 func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 25
-	err = en.Append(0xdc, 0x0, 0x19)
+	// array header, size 26
+	err = en.Append(0xdc, 0x0, 0x1a)
 	if err != nil {
 		return
 	}
@@ -870,14 +875,19 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Idx")
 		return
 	}
+	err = en.WriteTime(z.DiskMTime)
+	if err != nil {
+		err = msgp.WrapError(err, "DiskMTime")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 25
-	o = append(o, 0xdc, 0x0, 0x19)
+	// array header, size 26
+	o = append(o, 0xdc, 0x0, 0x1a)
 	o = msgp.AppendString(o, z.Volume)
 	o = msgp.AppendString(o, z.Name)
 	o = msgp.AppendString(o, z.VersionID)
@@ -922,6 +932,7 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendTime(o, z.SuccessorModTime)
 	o = msgp.AppendBool(o, z.Fresh)
 	o = msgp.AppendInt(o, z.Idx)
+	o = msgp.AppendTime(o, z.DiskMTime)
 	return
 }
 
@@ -933,8 +944,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 25 {
-		err = msgp.ArrayError{Wanted: 25, Got: zb0001}
+	if zb0001 != 26 {
+		err = msgp.ArrayError{Wanted: 26, Got: zb0001}
 		return
 	}
 	z.Volume, bts, err = msgp.ReadStringBytes(bts)
@@ -1099,6 +1110,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "Idx")
 		return
 	}
+	z.DiskMTime, bts, err = msgp.ReadTimeBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "DiskMTime")
+		return
+	}
 	o = bts
 	return
 }
@@ -1116,7 +1132,7 @@ func (z *FileInfo) Msgsize() (s int) {
 	for za0003 := range z.Parts {
 		s += z.Parts[za0003].Msgsize()
 	}
-	s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize + msgp.IntSize
+	s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize + msgp.IntSize + msgp.TimeSize
 	return
 }
 

cmd/storage-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v42" // Added FreeVersions to FileInfoVersions
+	storageRESTVersion       = "v43" // Added DiskMTime field for FileInfo
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )

cmd/sts-handlers.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"crypto/x509"
 	"encoding/base64"
 	"errors"
@@ -245,22 +246,18 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	}
 
 	m := map[string]interface{}{
-		expClaim: UTCNow().Add(duration).Unix(),
+		expClaim:    UTCNow().Add(duration).Unix(),
+		parentClaim: user.AccessKey,
 	}
 
-	policies, err := globalIAMSys.PolicyDBGet(user.AccessKey, false)
+	// Validate that user.AccessKey's policies can be retrieved - it may not
+	// be in case the user is disabled.
+	_, err = globalIAMSys.PolicyDBGet(user.AccessKey, false)
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
-	policyName := strings.Join(policies, ",")
-
-	// This policy is the policy associated with the user
-	// requesting for temporary credentials. The temporary
-	// credentials will inherit the same policy requirements.
-	m[iamPolicyClaimNameOpenID()] = policyName
-
 	if len(sessionPolicyStr) > 0 {
 		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
 	}
@@ -272,12 +269,12 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Set the parent of the temporary access key, this is useful
-	// in obtaining service accounts by this cred.
+	// Set the parent of the temporary access key, so that it's access
+	// policy is inherited from `user.AccessKey`.
 	cred.ParentUser = user.AccessKey
 
 	// Set the newly generated credentials.
-	if err = globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, policyName); err != nil {
+	if err = globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, ""); err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
 	}
@@ -483,7 +480,17 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		issFromToken, _ = v.(string)
 	}
 
-	cred.ParentUser = "openid:" + subFromToken + ":" + issFromToken
+	// Since issFromToken can have `/` characters (it is typically the
+	// provider URL), we hash and encode it to base64 here. This is needed
+	// because there will be a policy mapping stored on drives whose
+	// filename is this parentUser: therefore, it needs to have only valid
+	// filename characters and needs to have bounded length.
+	{
+		h := sha256.New()
+		h.Write([]byte("openid:" + subFromToken + ":" + issFromToken))
+		bs := h.Sum(nil)
+		cred.ParentUser = base64.RawURLEncoding.EncodeToString(bs)
+	}
 
 	// Set the newly generated credentials.
 	if err = globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, policyName); err != nil {
@@ -787,12 +794,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	parentUser := "tls:" + certificate.Subject.CommonName
 
 	tmpCredentials, err := auth.GetNewCredentialsWithMetadata(map[string]interface{}{
-		expClaim:                   UTCNow().Add(expiry).Unix(),
-		parentClaim:                parentUser,
-		subClaim:                   certificate.Subject.CommonName,
-		audClaim:                   certificate.Subject.Organization,
-		issClaim:                   certificate.Issuer.CommonName,
-		iamPolicyClaimNameOpenID(): certificate.Subject.CommonName,
+		expClaim:    UTCNow().Add(expiry).Unix(),
+		parentClaim: parentUser,
+		subClaim:    certificate.Subject.CommonName,
+		audClaim:    certificate.Subject.Organization,
+		issClaim:    certificate.Issuer.CommonName,
 	}, globalActiveCred.SecretKey)
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)

cmd/sts-handlers_test.go

@@ -31,12 +31,16 @@ import (
 	"github.com/minio/madmin-go"
 	minio "github.com/minio/minio-go/v7"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"golang.org/x/oauth2"
 )
 
 func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
 	suite.SetUpSuite(c)
+	// The STS for root test needs to be the first one after setup.
+	suite.TestSTSForRoot(c)
 	suite.TestSTS(c)
+	suite.TestSTSWithGroupPolicy(c)
 	suite.TearDownSuite(c)
 }
 
@@ -154,8 +158,176 @@ func (s *TestSuiteIAM) TestSTS(c *check) {
 	}
 }
 
-func (s *TestSuiteIAM) GetLDAPServer(c *check) string {
-	return os.Getenv(EnvTestLDAPServer)
+func (s *TestSuiteIAM) TestSTSWithGroupPolicy(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create policy, user and associate policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	// confirm that the user is unable to access the bucket - we have not
+	// yet set any policy
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
+		Group:   "test-group",
+		Members: []string{accessKey},
+	})
+	if err != nil {
+		c.Fatalf("unable to add user to group: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, policy, "test-group", true)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	// confirm that the user is able to access the bucket - permission comes
+	// from group.
+	c.mustListObjects(ctx, uClient, bucket)
+
+	// Create STS user.
+	assumeRole := cr.STSAssumeRole{
+		Client:      s.TestSuiteCommon.client,
+		STSEndpoint: s.endPoint,
+		Options: cr.STSAssumeRoleOptions{
+			AccessKey: accessKey,
+			SecretKey: secretKey,
+			Location:  "",
+		},
+	}
+	value, err := assumeRole.Retrieve()
+	if err != nil {
+		c.Fatalf("err calling assumeRole: %v", err)
+	}
+
+	// Check that STS user client has access coming from parent user's
+	// group.
+	minioClient, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("Error initializing client: %v", err)
+	}
+
+	// Validate that the client from sts creds can access the bucket.
+	c.mustListObjects(ctx, minioClient, bucket)
+
+	// Validate that the client cannot remove any objects
+	err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{})
+	if err.Error() != "Access Denied." {
+		c.Fatalf("unexpected non-access-denied err: %v", err)
+	}
+}
+
+// TestSTSForRoot - needs to be the first test after server setup due to the
+// buckets list check.
+func (s *TestSuiteIAM) TestSTSForRoot(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket create error: %v", err)
+	}
+
+	assumeRole := cr.STSAssumeRole{
+		Client:      s.TestSuiteCommon.client,
+		STSEndpoint: s.endPoint,
+		Options: cr.STSAssumeRoleOptions{
+			AccessKey: globalActiveCred.AccessKey,
+			SecretKey: globalActiveCred.SecretKey,
+			Location:  "",
+		},
+	}
+
+	value, err := assumeRole.Retrieve()
+	if err != nil {
+		c.Fatalf("err calling assumeRole: %v", err)
+	}
+
+	minioClient, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("Error initializing client: %v", err)
+	}
+
+	// Validate that the client from sts creds can access the bucket.
+	c.mustListObjects(ctx, minioClient, bucket)
+
+	// Validate that a bucket can be created
+	bucket2 := getRandomBucketName()
+	err = minioClient.MakeBucket(ctx, bucket2, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Validate that admin APIs can be called - create an madmin client with
+	// user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	accInfo, err := userAdmClient.AccountInfo(ctx, madmin.AccountOpts{})
+	if err != nil {
+		c.Fatalf("root user STS should be able to get account info: %v", err)
+	}
+
+	gotBuckets := set.NewStringSet()
+	for _, b := range accInfo.Buckets {
+		gotBuckets.Add(b.Name)
+		if !(b.Access.Read && b.Access.Write) {
+			c.Fatalf("root user should have read and write access to bucket: %v", b.Name)
+		}
+	}
+	shouldHaveBuckets := set.CreateStringSet(bucket2, bucket)
+	if !gotBuckets.Equals(shouldHaveBuckets) {
+		c.Fatalf("root user should have access to all buckets")
+	}
 }
 
 // SetUpLDAP - expects to setup an LDAP test server using the test LDAP
@@ -277,7 +449,13 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
 		c.Fatalf("Expected to fail to create STS cred with no associated policy!")
 	}
 
+	// Attempting to set a non-existent policy should fail.
 	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
+	err = s.adm.SetPolicy(ctx, policy+"x", userDN, false)
+	if err == nil {
+		c.Fatalf("should not be able to set non-existent policy")
+	}
+
 	err = s.adm.SetPolicy(ctx, policy, userDN, false)
 	if err != nil {
 		c.Fatalf("Unable to set policy: %v", err)

cmd/typed-errors.go

@@ -84,6 +84,10 @@ var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
 // error returned when policy to be deleted is in use.
 var errPolicyInUse = errors.New("Specified policy is in use and cannot be deleted.")
 
+// error returned when more than a single policy is specified when only one is
+// expectd.
+var errTooManyPolicies = errors.New("Only a single policy may be specified here.")
+
 // error returned in IAM subsystem when an external users systems is configured.
 var errIAMActionNotAllowed = errors.New("Specified IAM action is not allowed")
 

cmd/utils_test.go

@@ -29,6 +29,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 )
 
 // Tests maximum object size.
@@ -486,3 +487,32 @@ func TestGetMinioMode(t *testing.T) {
 	testMinioMode(globalMinioModeGatewayPrefix + globalGatewayName)
 
 }
+
+func TestTimedValue(t *testing.T) {
+	var cache timedValue
+	t.Parallel()
+	cache.Once.Do(func() {
+		cache.TTL = 2 * time.Second
+		cache.Update = func() (interface{}, error) {
+			return time.Now(), nil
+		}
+	})
+
+	i, _ := cache.Get()
+	t1 := i.(time.Time)
+
+	j, _ := cache.Get()
+	t2 := j.(time.Time)
+
+	if !t1.Equal(t2) {
+		t.Fatalf("expected time to be equal: %s != %s", t1, t2)
+	}
+
+	time.Sleep(3 * time.Second)
+	k, _ := cache.Get()
+	t3 := k.(time.Time)
+
+	if t1.Equal(t3) {
+		t.Fatalf("expected time to be un-equal: %s == %s", t1, t3)
+	}
+}

cmd/xl-storage.go

@@ -375,32 +375,39 @@ func (s *xlStorage) Healing() *healingTracker {
 	return &h
 }
 
-func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte, error) {
+// readsMetadata and returns disk mTime information for xl.meta
+func (s *xlStorage) readMetadataWithDMTime(ctx context.Context, itemPath string) ([]byte, time.Time, error) {
 	if contextCanceled(ctx) {
-		return nil, ctx.Err()
+		return nil, time.Time{}, ctx.Err()
 	}
 
 	if err := checkPathLength(itemPath); err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
 
 	f, err := OpenFile(itemPath, readMode, 0)
 	if err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	defer f.Close()
 	stat, err := f.Stat()
 	if err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if stat.IsDir() {
-		return nil, &os.PathError{
+		return nil, time.Time{}, &os.PathError{
 			Op:   "open",
 			Path: itemPath,
 			Err:  syscall.EISDIR,
 		}
 	}
-	return readXLMetaNoData(f, stat.Size())
+	buf, err := readXLMetaNoData(f, stat.Size())
+	return buf, stat.ModTime().UTC(), err
+}
+
+func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte, error) {
+	buf, _, err := s.readMetadataWithDMTime(ctx, itemPath)
+	return buf, err
 }
 
 func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
@@ -1214,11 +1221,18 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 	if err != nil {
 		return fi, err
 	}
+	// Validate file path length, before reading.
+	filePath := pathJoin(volumeDir, path)
+	if err = checkPathLength(filePath); err != nil {
+		return fi, err
+	}
+
 	var buf []byte
+	var dmTime time.Time
 	if readData {
-		buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
+		buf, dmTime, err = s.readAllData(ctx, volumeDir, pathJoin(filePath, xlStorageFormatFile))
 	} else {
-		buf, err = s.readMetadata(ctx, pathJoin(volumeDir, path, xlStorageFormatFile))
+		buf, dmTime, err = s.readMetadataWithDMTime(ctx, pathJoin(filePath, xlStorageFormatFile))
 		if err != nil {
 			if osIsNotExist(err) {
 				if aerr := Access(volumeDir); aerr != nil && osIsNotExist(aerr) {
@@ -1231,7 +1245,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 
 	if err != nil {
 		if err == errFileNotFound {
-			buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFileV1))
+			buf, dmTime, err = s.readAllData(ctx, volumeDir, pathJoin(filePath, xlStorageFormatFileV1))
 			if err != nil {
 				if err == errFileNotFound {
 					if versionID != "" {
@@ -1257,6 +1271,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 	if err != nil {
 		return fi, err
 	}
+	fi.DiskMTime = dmTime
 
 	if len(fi.Data) == 0 {
 		// We did not read inline data, so we have no references.
@@ -1299,7 +1314,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 			len(fi.Parts) == 1 {
 			partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
 			dataPath := pathJoin(volumeDir, path, fi.DataDir, partPath)
-			fi.Data, err = s.readAllData(volumeDir, dataPath)
+			fi.Data, _, err = s.readAllData(ctx, volumeDir, dataPath)
 			if err != nil {
 				return FileInfo{}, err
 			}
@@ -1309,38 +1324,42 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 	return fi, nil
 }
 
-func (s *xlStorage) readAllData(volumeDir string, filePath string) (buf []byte, err error) {
+func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath string) (buf []byte, dmTime time.Time, err error) {
+	if contextCanceled(ctx) {
+		return nil, time.Time{}, ctx.Err()
+	}
+
 	f, err := OpenFileDirectIO(filePath, readMode, 0666)
 	if err != nil {
 		if osIsNotExist(err) {
 			// Check if the object doesn't exist because its bucket
 			// is missing in order to return the correct error.
 			if err = Access(volumeDir); err != nil && osIsNotExist(err) {
-				return nil, errVolumeNotFound
+				return nil, dmTime, errVolumeNotFound
 			}
-			return nil, errFileNotFound
+			return nil, dmTime, errFileNotFound
 		} else if osIsPermission(err) {
-			return nil, errFileAccessDenied
+			return nil, dmTime, errFileAccessDenied
 		} else if isSysErrNotDir(err) || isSysErrIsDir(err) {
-			return nil, errFileNotFound
+			return nil, dmTime, errFileNotFound
 		} else if isSysErrHandleInvalid(err) {
 			// This case is special and needs to be handled for windows.
-			return nil, errFileNotFound
+			return nil, dmTime, errFileNotFound
 		} else if isSysErrIO(err) {
-			return nil, errFaultyDisk
+			return nil, dmTime, errFaultyDisk
 		} else if isSysErrTooManyFiles(err) {
-			return nil, errTooManyOpenFiles
+			return nil, dmTime, errTooManyOpenFiles
 		} else if isSysErrInvalidArg(err) {
 			st, _ := Lstat(filePath)
 			if st != nil && st.IsDir() {
 				// Linux returns InvalidArg for directory O_DIRECT
 				// we need to keep this fallback code to return correct
 				// errors upwards.
-				return nil, errFileNotFound
+				return nil, dmTime, errFileNotFound
 			}
-			return nil, errUnsupportedDisk
+			return nil, dmTime, errUnsupportedDisk
 		}
-		return nil, err
+		return nil, dmTime, err
 	}
 	r := &xioutil.ODirectReader{
 		File:      f,
@@ -1352,10 +1371,10 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string) (buf []byte,
 	stat, err := f.Stat()
 	if err != nil {
 		buf, err = ioutil.ReadAll(r)
-		return buf, osErrToFileErr(err)
+		return buf, dmTime, osErrToFileErr(err)
 	}
 	if stat.IsDir() {
-		return nil, errFileNotFound
+		return nil, dmTime, errFileNotFound
 	}
 
 	// Read into appropriate buffer.
@@ -1369,7 +1388,7 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string) (buf []byte,
 	// Read file...
 	_, err = io.ReadFull(r, buf)
 
-	return buf, osErrToFileErr(err)
+	return buf, stat.ModTime().UTC(), osErrToFileErr(err)
 }
 
 // ReadAll reads from r until an error or EOF and returns the data it read.
@@ -1390,7 +1409,8 @@ func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (bu
 		return nil, err
 	}
 
-	return s.readAllData(volumeDir, filePath)
+	buf, _, err = s.readAllData(ctx, volumeDir, filePath)
+	return buf, err
 }
 
 // ReadFile reads exactly len(buf) bytes into buf. It returns the
@@ -1962,7 +1982,10 @@ func (s *xlStorage) Delete(ctx context.Context, volume string, path string, recu
 func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
 	defer func() {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, fmt.Errorf("srcVolume: %s, srcPath: %s, dstVolume: %s:, dstPath: %s - error %v",
+				srcVolume, srcPath,
+				dstVolume, dstPath,
+				err))
 		}
 		if err == nil {
 			if s.globalSync {

dockerscripts/docker-entrypoint.sh

@@ -8,79 +8,6 @@ if [ "${1}" != "minio" ]; then
     fi
 fi
 
-## look for specific a `config.env` file to load all the
-## minio settings from
-docker_minio_env() {
-    if [ -f "$MINIO_CONFIG_ENV_FILE" ]; then
-        config_env_file="${MINIO_CONFIG_ENV_FILE}"
-    else
-        config_env_file="/run/secrets/${MINIO_CONFIG_ENV_FILE}"
-    fi
-    if [ -f "$config_env_file" ]; then
-        # shellcheck source=/dev/null
-        . "${config_env_file}"
-    fi
-}
-
-## Look for docker secrets at given absolute path or in default documented location.
-docker_secrets_env_old() {
-    if [ -f "$MINIO_ACCESS_KEY_FILE" ]; then
-        ACCESS_KEY_FILE="$MINIO_ACCESS_KEY_FILE"
-    else
-        ACCESS_KEY_FILE="/run/secrets/$MINIO_ACCESS_KEY_FILE"
-    fi
-    if [ -f "$MINIO_SECRET_KEY_FILE" ]; then
-        SECRET_KEY_FILE="$MINIO_SECRET_KEY_FILE"
-    else
-        SECRET_KEY_FILE="/run/secrets/$MINIO_SECRET_KEY_FILE"
-    fi
-
-    if [ -f "$ACCESS_KEY_FILE" ]; then
-        MINIO_ACCESS_KEY="$(cat "$ACCESS_KEY_FILE")"
-        export MINIO_ACCESS_KEY
-    fi
-    if [ -f "$SECRET_KEY_FILE" ]; then
-        MINIO_SECRET_KEY="$(cat "$SECRET_KEY_FILE")"
-        export MINIO_SECRET_KEY
-    fi
-}
-
-docker_secrets_env() {
-    if [ -f "$MINIO_ROOT_USER_FILE" ]; then
-        ROOT_USER_FILE="$MINIO_ROOT_USER_FILE"
-    else
-        ROOT_USER_FILE="/run/secrets/$MINIO_ROOT_USER_FILE"
-    fi
-    if [ -f "$MINIO_ROOT_PASSWORD_FILE" ]; then
-        ROOT_PASSWORD_FILE="$MINIO_ROOT_PASSWORD_FILE"
-    else
-        ROOT_PASSWORD_FILE="/run/secrets/$MINIO_ROOT_PASSWORD_FILE"
-    fi
-
-    if [ -f "$ROOT_USER_FILE" ]; then
-        MINIO_ROOT_USER="$(cat "$ROOT_USER_FILE")"
-        export MINIO_ROOT_USER
-    fi
-    if [ -f "$ROOT_PASSWORD_FILE" ]; then
-        MINIO_ROOT_PASSWORD="$(cat "$ROOT_PASSWORD_FILE")"
-        export MINIO_ROOT_PASSWORD
-    fi
-}
-
-## Set KMS_SECRET_KEY from docker secrets if provided
-docker_kms_secret_encryption_env() {
-    if [ -f "$MINIO_KMS_SECRET_KEY_FILE" ]; then
-        KMS_SECRET_KEY_FILE="$MINIO_KMS_SECRET_KEY_FILE"
-    else
-        KMS_SECRET_KEY_FILE="/run/secrets/$MINIO_KMS_SECRET_KEY_FILE"
-    fi
-
-    if [ -f "$KMS_SECRET_KEY_FILE" ]; then
-        MINIO_KMS_SECRET_KEY="$(cat "$KMS_SECRET_KEY_FILE")"
-        export MINIO_KMS_SECRET_KEY
-    fi
-}
-
 # su-exec to requested user, if service cannot run exec will fail.
 docker_switch_user() {
     if [ -n "${MINIO_USERNAME}" ] && [ -n "${MINIO_GROUPNAME}" ]; then
@@ -98,19 +25,5 @@ docker_switch_user() {
     fi
 }
 
-## Set access env from secrets if necessary. Legacy
-docker_secrets_env_old
-
-## Set access env from secrets if necessary. Override
-docker_secrets_env
-
-## Set kms encryption from secrets if necessary. Override
-docker_kms_secret_encryption_env
-
-## Set all config environment variables from 'config.env' if necessary.
-## Overrides all previous settings and also overrides all
-## environment values passed from 'podman run -e ENV=value'
-docker_minio_env
-
 ## Switch to user if applicable.
 docker_switch_user "$@"

docs/bucket/lifecycle/README.md

@@ -82,27 +82,52 @@ e.g., To scan objects stored under `user-uploads/` prefix and remove versions ol
 ```
 
 
-### 3.2 Automatic removal of noncurrent versions older than most recent 
+### 3.2 Automatic removal of noncurrent versions keeping only most recent ones after noncurrent days
 
-It is possible to configure automatic removal of noncurrent versions older than the most recent `N` using MinIO specific lifecycle policy extension `MaxNoncurrentVersions`.
+It is possible to configure automatic removal of older noncurrent versions keeping only the most recent `N` using `NewerNoncurrentVersions`.
 
-e.g, To remove noncurrent versions of all objects older than most recent 5 noncurrent versions under the prefix `user-uploads/`,
+e.g, To remove noncurrent versions of all objects keeping the most recent 5 noncurrent versions under the prefix `user-uploads/` 30 days after they become noncurrent ,
 ```
 {
     "Rules": [
         {
-            "ID": "Remove noncurrent versions older than",
+            "ID": "Keep only most recent 5 noncurrent versions",
             "Status": "Enabled",
             "Filter": {
                 "Prefix": "users-uploads/"
             },
             "NoncurrentVersionExpiration": {
-                "MaxNoncurrentVersions": 5
+                "NewerNoncurrentVersions": 5,
+                "NoncurrentDays": 30
             }
         }
     ]
 }
 ```
+
+#### 3.2.a Automatic removal of noncurrent versions keeping only most recent ones immediately (MinIO only extension)
+
+This is available only on MinIO as an extension to the NewerNoncurrentVersions feature. The following rule makes it possible to remove older noncurrent versions
+of objects under the prefix `user-uploads/` as soon as there are more than `N` noncurrent versions of an object.
+
+```
+{
+    "Rules": [
+        {
+            "ID": "Keep only most recent 5 noncurrent versions",
+            "Status": "Enabled",
+            "Filter": {
+                "Prefix": "users-uploads/"
+            },
+            "NoncurrentVersionExpiration": {
+                "NewerNoncurrentVersions": 5
+            }
+        }
+    ]
+}
+```
+Note: This rule has an implicit zero NoncurrentDays, which makes the expiry of those 'extra' noncurrent versions immediate.
+
 ### 3.3 Automatic removal of delete markers with no other versions
 
 When an object has only one version as a delete marker, the latter can be automatically removed after a certain number of days using the following configuration:

docs/bucket/replication/setup_3site_replication.sh

@@ -240,18 +240,18 @@ head -c 221227088 </dev/urandom >200M
 sleep 10
 
 echo "Verifying ETag for all objects"
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket
 
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket
-./s3-check-md5 -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket
+./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket
 
 catch

docs/debugging/hash-set/main.go

@@ -0,0 +1,99 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"hash/crc32"
+	"log"
+
+	"github.com/dchest/siphash"
+	"github.com/google/uuid"
+)
+
+// hashes the key returning an integer based on the input algorithm.
+// This function currently supports
+// - SIPMOD
+func sipHashMod(key string, cardinality int, id [16]byte) int {
+	if cardinality <= 0 {
+		return -1
+	}
+	// use the faster version as per siphash docs
+	// https://github.com/dchest/siphash#usage
+	k0, k1 := binary.LittleEndian.Uint64(id[0:8]), binary.LittleEndian.Uint64(id[8:16])
+	sum64 := siphash.Hash(k0, k1, []byte(key))
+	return int(sum64 % uint64(cardinality))
+}
+
+// hashOrder - hashes input key to return consistent
+// hashed integer slice. Returned integer order is salted
+// with an input key. This results in consistent order.
+// NOTE: collisions are fine, we are not looking for uniqueness
+// in the slices returned.
+func hashOrder(key string, cardinality int) []int {
+	if cardinality <= 0 {
+		// Returns an empty int slice for cardinality < 0.
+		return nil
+	}
+
+	nums := make([]int, cardinality)
+	keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable)
+
+	start := int(keyCrc % uint32(cardinality))
+	for i := 1; i <= cardinality; i++ {
+		nums[i-1] = 1 + ((start + i) % cardinality)
+	}
+	return nums
+}
+
+var (
+	object, deploymentID string
+	setCount, shards     int
+)
+
+func main() {
+	flag.StringVar(&object, "object", "", "Select an object")
+	flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'")
+	flag.IntVar(&setCount, "set-count", 0, "Total set count")
+	flag.IntVar(&shards, "shards", 0, "Total shards count")
+
+	flag.Parse()
+
+	if object == "" {
+		log.Fatalln("object name is mandatory")
+	}
+
+	if deploymentID == "" {
+		log.Fatalln("deployment ID is mandatory")
+	}
+
+	if setCount == 0 {
+		log.Fatalln("set count cannot be zero")
+	}
+
+	if shards == 0 {
+		log.Fatalln("total shards cannot be zero")
+	}
+
+	id := uuid.MustParse(deploymentID)
+
+	fmt.Println("Erasure distribution for the object", hashOrder(object, shards))
+	fmt.Println("Erasure setNumber for the object", sipHashMod(object, setCount, id)+1)
+}

docs/debugging/s3-check-md5/main.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"log"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 
@@ -35,6 +36,8 @@ import (
 var (
 	endpoint, accessKey, secretKey string
 	bucket, prefix                 string
+	debug                          bool
+	versions                       bool
 )
 
 // getMD5Sum returns MD5 sum of given data.
@@ -50,6 +53,8 @@ func main() {
 	flag.StringVar(&secretKey, "secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key")
 	flag.StringVar(&bucket, "bucket", "", "Select a specific bucket")
 	flag.StringVar(&prefix, "prefix", "", "Select a prefix")
+	flag.BoolVar(&debug, "debug", false, "Prints HTTP network calls to S3 endpoint")
+	flag.BoolVar(&versions, "versions", false, "Verify all versions")
 	flag.Parse()
 
 	if endpoint == "" {
@@ -81,7 +86,9 @@ func main() {
 		log.Fatalln()
 	}
 
-	// s3Client.TraceOn(os.Stderr)
+	if debug {
+		s3Client.TraceOn(os.Stderr)
+	}
 
 	var buckets []string
 	if bucket != "" {
@@ -100,7 +107,8 @@ func main() {
 		opts := minio.ListObjectsOptions{
 			Recursive:    true,
 			Prefix:       prefix,
-			WithVersions: true,
+			WithVersions: versions,
+			WithMetadata: true,
 		}
 
 		// List all objects from a bucket-name with a matching prefix.
@@ -110,9 +118,19 @@ func main() {
 				continue
 			}
 			if object.IsDeleteMarker {
+				log.Println("DELETE marker skipping object:", object.Key)
+				continue
+			}
+			if _, ok := object.UserMetadata["X-Amz-Server-Side-Encryption-Customer-Algorithm"]; ok {
+				log.Println("Objects encrypted with SSE-C do not have md5sum as ETag:", object.Key)
+				continue
+			}
+			if v, ok := object.UserMetadata["X-Amz-Server-Side-Encryption"]; ok && v == "aws:kms" {
+				log.Println("Objects encrypted with SSE-KMS do not have md5sum as ETag:", object.Key)
 				continue
 			}
 			parts := 1
+			multipart := false
 			s := strings.Split(object.ETag, "-")
 			switch len(s) {
 			case 1:
@@ -121,39 +139,41 @@ func main() {
 				if p, err := strconv.Atoi(s[1]); err == nil {
 					parts = p
 				} else {
-					log.Fatalln("ETAG: wrong format:", err)
+					log.Println("ETAG: wrong format:", err)
 					continue
 				}
+				multipart = true
 			default:
-				log.Fatalln("Unexpected ETAG format", object.ETag)
+				log.Println("Unexpected ETAG format", object.ETag)
+				continue
 			}
 
 			var partsMD5Sum [][]byte
-
 			for p := 1; p <= parts; p++ {
-				obj, err := s3Client.GetObject(context.Background(), bucket, object.Key,
-					minio.GetObjectOptions{VersionID: object.VersionID, PartNumber: p})
+				opts := minio.GetObjectOptions{
+					VersionID:  object.VersionID,
+					PartNumber: p,
+				}
+				obj, err := s3Client.GetObject(context.Background(), bucket, object.Key, opts)
 				if err != nil {
-					log.Fatalln("GET", bucket, object.Key, object.VersionID, "=>", err)
+					log.Println("GET", bucket, object.Key, object.VersionID, "=>", err)
 					continue
 				}
 				h := md5.New()
 				if _, err := io.Copy(h, obj); err != nil {
-					log.Fatalln("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err)
+					log.Println("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err)
 					continue
 				}
 				partsMD5Sum = append(partsMD5Sum, h.Sum(nil))
 			}
 
 			corrupted := false
-
-			switch parts {
-			case 1:
+			if !multipart {
 				md5sum := fmt.Sprintf("%x", partsMD5Sum[0])
 				if md5sum != object.ETag {
 					corrupted = true
 				}
-			default:
+			} else {
 				var totalMD5SumBytes []byte
 				for _, sum := range partsMD5Sum {
 					totalMD5SumBytes = append(totalMD5SumBytes, sum...)
@@ -165,10 +185,10 @@ func main() {
 			}
 
 			if corrupted {
-				log.Fatalln("CORRUPTED object:", bucket, object.Key, object.VersionID)
+				log.Println("CORRUPTED object:", bucket, object.Key, object.VersionID)
+			} else {
+				log.Println("INTACT object:", bucket, object.Key, object.VersionID)
 			}
-
-			log.Println("INTACT", bucket, object.Key, object.VersionID)
 		}
 	}
 }

docs/disk-caching/DESIGN.md

@@ -87,7 +87,7 @@ Disk caching caches objects for **downloaded** objects i.e
 - Cache continues to work for read-only operations such as GET, HEAD when backend is offline.
 - Cache-Control and Expires headers can be used to control how long objects stay in the cache. ETag of cached objects are not validated with backend until expiry time as per the Cache-Control or Expires header is met.
 - All range GET requests are cached by default independently, this may be not desirable in all situations when cache storage is limited and where downloading an entire object at once might be more optimal. To optionally turn this feature off, and allow downloading entire object in the background `export MINIO_CACHE_RANGE=off`.
-- To ensure security guarantees, encrypted objects are normally not cached. However, if you wish to encrypt cached content on disk, you can set MINIO_CACHE_ENCRYPTION_MASTER_KEY environment variable to set a cache KMS
+- To ensure security guarantees, encrypted objects are normally not cached. However, if you wish to encrypt cached content on disk, you can set MINIO_CACHE_ENCRYPTION_SECRET_KEY environment variable to set a cache KMS
 master key to automatically encrypt all cached content.
 
   Note that cache KMS master key is not recommended for use in production deployments. If the MinIO server/gateway machine is ever compromised, the cache KMS master key must also be treated as compromised.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2021-12-09T06-19-41Z
+  image: quay.io/minio/minio:RELEASE.2021-12-27T07-23-18Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/orchestration/kubernetes/README.md

@@ -8,7 +8,7 @@ There are multiple options to deploy MinIO on Kubernetes:
 
 - MinIO-Operator: Operator offers seamless way to create and update highly available distributed MinIO clusters. Refer [MinIO Operator documentation](https://github.com/minio/minio-operator/blob/master/README.md) for more details.
 
-- Helm Chart: MinIO Helm Chart offers customizable and easy MinIO deployment with a single command. Refer [MinIO Helm Chart documentation](https://github.com/minio/charts) for more details.
+- Helm Chart: MinIO Helm Chart offers customizable and easy MinIO deployment with a single command. Refer [MinIO Helm Chart documentation](https://github.com/minio/minio/tree/master/helm/minio) for more details.
 
 ## Monitoring MinIO in Kubernetes
 

docs/site-replication/run-multi-site.sh

@@ -79,26 +79,26 @@ if [ $? -eq 0 ]; then
 fi
 
 ./mc admin user info minio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "policy mapping missing, exiting.."
     exit_1;
 fi
 
 ./mc admin user info minio2 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "policy mapping missing, exiting.."
     exit_1;
 fi
 
 ./mc admin user info minio3 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "policy mapping missing, exiting.."
     exit_1;
 fi
 
 # LDAP simple user
 ./mc admin user svcacct add minio2 dillon --access-key testsvc --secret-key testsvc123
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "adding svc account failed, exiting.."
     exit_1;
 fi
@@ -106,19 +106,19 @@ fi
 sleep 10
 
 ./mc admin user svcacct info minio1 testsvc
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "svc account not mirrored, exiting.."
     exit_1;
 fi
 
 ./mc admin user svcacct info minio2 testsvc
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "svc account not mirrored, exiting.."
     exit_1;
 fi
 
 ./mc admin user svcacct rm minio1 testsvc
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "removing svc account failed, exiting.."
     exit_1;
 fi
@@ -140,13 +140,13 @@ fi
 
 sleep 5
 ./mc stat minio2/newbucket
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "expecting bucket to be present. exiting.."
     exit_1;
 fi
 
 ./mc stat minio3/newbucket
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "expecting bucket to be present. exiting.."
     exit_1;
 fi
@@ -155,13 +155,13 @@ fi
 
 sleep 5
 ./mc stat minio1/newbucket/README.md
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "expecting object to be present. exiting.."
     exit_1;
 fi
 
 ./mc stat minio3/newbucket/README.md
-if [ $? -eq 1 ]; then
+if [ $? -ne 0 ]; then
     echo "expecting object to be present. exiting.."
     exit_1;
 fi
@@ -180,3 +180,28 @@ if [ $? -eq 0 ]; then
     echo "expected file to be deleted, exiting.."
     exit_1;
 fi
+
+./mc mb --with-lock minio3/newbucket-olock
+sleep 5
+
+enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio2}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi
+
+enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio1}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi

docs/sts/keycloak.md

@@ -4,7 +4,8 @@ Keycloak is an open source Identity and Access Management solution aimed at mode
 
 ## Prerequisites
 
-Configure and install keycloak server by following [Keycloak Installation Guide](https://www.keycloak.org/docs/latest/getting_started/index.html) (finish upto section 3.4)
+Configure and install keycloak server by following [Keycloak Installation Guide](https://www.keycloak.org/docs/latest/server_installation/#installing-the-software).
+For a quick installation, docker-compose reference configs are also available on the [Keycloak GitHub](https://github.com/keycloak/keycloak-containers/tree/main/docker-compose-examples). 
 
 ### Configure Keycloak Realm
 - Go to Clients
@@ -27,7 +28,7 @@ Configure and install keycloak server by following [Keycloak Installation Guide]
   - Save
 
 - Go to Clients
-  - Client on `account`
+  - Click on `account`
   - Mappers
   - Create
     - `Name` with any text
@@ -37,7 +38,7 @@ Configure and install keycloak server by following [Keycloak Installation Guide]
     - `Claim JSON Type` is `string`
   - Save
 
-- Open http://localhost:8080/auth/realms/minio/.well-known/openid-configuration to verify OpenID discovery document, verify it has `authorization_endpoint` and `jwks_uri`
+- Open http://localhost:8080/auth/realms/{your-realm-name}/.well-known/openid-configuration to verify OpenID discovery document, verify it has `authorization_endpoint` and `jwks_uri`
 
 ### Enable Keycloak Admin REST API support
 Before being able to authenticate against the Admin REST API using a client_id and a client_secret you need to make sure the client is configured as it follows:
@@ -116,7 +117,7 @@ MINIO_IDENTITY_OPENID_COMMENT       (sentence)  optionally add a comment to this
 
 Set `identity_openid` config with `config_url`, `client_id` and restart MinIO
 ```
-~ mc admin config set myminio identity_openid config_url="http://localhost:8080/auth/realms/minio/.well-known/openid-configuration" client_id="account"
+~ mc admin config set myminio identity_openid config_url="http://localhost:8080/auth/realms/{your-realm-name}/.well-known/openid-configuration" client_id="account"
 ```
 > NOTE: You can configure the `scopes` parameter to restrict the OpenID scopes requested by minio to the IdP, for example, `"openid,policy_role_attribute"`, being `policy_role_attribute` a client_scope / client_mapper that maps a role attribute called policy to a `policy` claim returned by Keycloak
 

docs/tls/README.md

@@ -148,7 +148,7 @@ DNS.1 = localhost
 Run `openssl` by specifying the configuration file and enter a passphrase if prompted:
 
 ```sh
-openssl req -new -x509 -nodes -days 730 -key private.key -out public.crt -config openssl.conf
+openssl req -new -x509 -nodes -days 730 -keyout private.key -out public.crt -config openssl.conf
 ```
 
 ### <a name="using-gnu-tls"></a>3.3 Use GnuTLS (for Windows) to Generate a Certificate

go.mod

@@ -45,16 +45,15 @@ require (
 	github.com/lib/pq v1.9.0
 	github.com/miekg/dns v1.1.43
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.12.8
+	github.com/minio/console v0.13.1
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.1.17
-	github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2 // indirect
-	github.com/minio/minio-go/v7 v7.0.17
+	github.com/minio/madmin-go v1.1.23
+	github.com/minio/minio-go/v7 v7.0.20
 	github.com/minio/parquet-go v1.1.0
-	github.com/minio/pkg v1.1.9
-	github.com/minio/selfupdate v0.3.1
+	github.com/minio/pkg v1.1.11
+	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.2.1
 	github.com/minio/sio v0.3.0
@@ -86,7 +85,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.19.1
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
 	golang.org/x/sys v0.0.0-20211020174200-9d6173849985
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
@@ -168,6 +167,7 @@ require (
 	github.com/minio/colorjson v1.0.1 // indirect
 	github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf // indirect
 	github.com/minio/filepath v1.0.0 // indirect
+	github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7 // indirect
 	github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7 // indirect
@@ -203,7 +203,7 @@ require (
 	go.mongodb.org/mongo-driver v1.4.6 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
-	golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect

go.sum

@@ -1071,8 +1071,8 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
 github.com/minio/colorjson v1.0.1/go.mod h1:oPM3zQQY8Gz9NGtgvuBEjQ+gPZLKAGc7T+kjMlwtOgs=
-github.com/minio/console v0.12.8 h1:CSJ54UC+il39cWiBqck3rSNiCfBIvIveUfvLtmQjN3E=
-github.com/minio/console v0.12.8/go.mod h1:SaJoJB4QCU5+jjSaizwQOXHzN78mrqidyBMa3ULm+wg=
+github.com/minio/console v0.13.1 h1:/T/k9DeAqcBfAz7e314WwXreN3iMSiOHgIFlKF//Q+E=
+github.com/minio/console v0.13.1/go.mod h1:Igh8I8l0O6pNmzOJVMSIk/ftoCt635LGUJ//ShzFXhw=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf h1:wylCc/PdvdTIqYqVNEU9LJAZBanvfGY1TwTnjM3zQaA=
@@ -1087,8 +1087,10 @@ github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
 github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
 github.com/minio/madmin-go v1.1.15/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
-github.com/minio/madmin-go v1.1.17 h1:P7my58nSulXawjixhS4S2pT43+3M+Y0P7M43I3YdzGo=
-github.com/minio/madmin-go v1.1.17/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
+github.com/minio/madmin-go v1.1.19/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
+github.com/minio/madmin-go v1.1.21/go.mod h1:vIDiJEjYOG27M/CgZPmxBdgdn3Yz5SIwtXtMzGAsqsA=
+github.com/minio/madmin-go v1.1.23 h1:VO/K+cFpX3HbNaLBEILd18V09ZpoSCE90522nHtmxdQ=
+github.com/minio/madmin-go v1.1.23/go.mod h1:wv8zCroSCnpjjQdmgsdJEkFH2oD4w9J40OZqbhxjiJ4=
 github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2 h1:xocb1RGyrDJ8PxkNn0NSbaBlfdU6J/Ag9QK62pb7nR8=
 github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2/go.mod h1:siI9jWTzj1KsNXgz6NOL/S7OTaAUM0OMi+zEkF08gnA=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -1097,8 +1099,9 @@ github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEp
 github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/minio-go/v7 v7.0.16-0.20211108161804-a7a36ee131df/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
-github.com/minio/minio-go/v7 v7.0.17 h1:5SiS3pqiQDbNhmXMxtqn2HzAInbN5cbHT7ip9F0F07E=
-github.com/minio/minio-go/v7 v7.0.17/go.mod h1:SyQ1IFeJuaa+eV5yEDxW7hYE1s5VVq5sgImDe27R+zg=
+github.com/minio/minio-go/v7 v7.0.18/go.mod h1:SyQ1IFeJuaa+eV5yEDxW7hYE1s5VVq5sgImDe27R+zg=
+github.com/minio/minio-go/v7 v7.0.20 h1:0+Xt1SkCKDgcx5cmo3UxXcJ37u5Gy+/2i/+eQYqmYJw=
+github.com/minio/minio-go/v7 v7.0.20/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
 github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7 h1:dkfuMNslMjGoJ4ArAMSoQhidYNdm3SgzLBP+f96O3/E=
 github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7/go.mod h1:lDpuz8nwsfhKlfiBaA3Z8AW019fWEAjO2gltfLbdorE=
 github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7 h1:vFtQqCt67ETp0JAkOKRWTKkgwFv14Vc1jJSxmQ8wJE0=
@@ -1109,10 +1112,12 @@ github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.11/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
 github.com/minio/pkg v1.1.3/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
-github.com/minio/pkg v1.1.9 h1:NJrcrQyFCSgyF+u6v7FbPXjjNV0oSnBuBevhsTKmA2U=
-github.com/minio/pkg v1.1.9/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
-github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
+github.com/minio/pkg v1.1.10/go.mod h1:3I8LLp1/HDhNKl35I8ve0mhzp7+bvVTrJmLqkdkOHME=
+github.com/minio/pkg v1.1.11 h1:FQMYrOZzZenQUVwvxM0YjoAZB15Wb+m3TZSu33Lynx0=
+github.com/minio/pkg v1.1.11/go.mod h1:2WJAxesjzmPK9MnLZKm5n1hVYfBg04f2GQs6N5ImNU8=
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
+github.com/minio/selfupdate v0.4.0 h1:A7t07pN4Ch1tBTIRStW0KhUVyykz+2muCqFsITQeEW8=
+github.com/minio/selfupdate v0.4.0/go.mod h1:mcDkzMgq8PRcpCRJo/NlPY7U45O5dfYl2Y0Rg7IustY=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
 github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
@@ -1634,8 +1639,9 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b h1:QAqMVf3pSa6eeTsuklijukjXBlj7Es2QQplab+/RbQ4=
+golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1738,8 +1744,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
-golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.3.1
-appVersion: RELEASE.2021-11-24T23-19-33Z
+version: 3.4.3
+appVersion: RELEASE.2021-12-20T22-07-16Z
 keywords:
   - minio
   - storage

helm/minio/templates/deployment.yaml

@@ -71,6 +71,9 @@ spec:
             - "-ce"
             - "/usr/bin/docker-entrypoint.sh minio server {{ $bucketRoot }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template "minio.extraArgs" . }}"
           volumeMounts:
+            - name: minio-user
+              mountPath: "/tmp/credentials"
+              readOnly: true
             {{- if .Values.persistence.enabled }}
             - name: export
               mountPath: {{ .Values.mountPath }}
@@ -99,6 +102,26 @@ spec:
             - name: MINIO_PROMETHEUS_AUTH_TYPE
               value: "public"
             {{- end}}
+            {{- if .Values.etcd.endpoints }}
+            - name: MINIO_ETCD_ENDPOINTS
+              value: {{ join "," .Values.etcd.endpoints | quote }}
+            {{- if .Values.etcd.clientCert }}
+            - name: MINIO_ETCD_CLIENT_CERT
+              value: "/tmp/credentials/etcd_client_cert.pem"
+            {{- end }}
+            {{- if .Values.etcd.clientCertKey }}
+            - name: MINIO_ETCD_CLIENT_CERT_KEY
+              value: "/tmp/credentials/etcd_client_cert_key.pem"
+            {{- end }}
+            {{- if .Values.etcd.pathPrefix }}
+            - name: MINIO_ETCD_PATH_PREFIX
+              value: {{ .Values.etcd.pathPrefix }}
+            {{- end }}
+            {{- if .Values.etcd.corednsPathPrefix }}
+            - name: MINIO_ETCD_COREDNS_PATH
+              value: {{ .Values.etcd.corednsPathPrefix }}
+            {{- end }}
+            {{- end }}
             {{- range $key, $val := .Values.environment }}
             - name: {{ $key }}
               value: {{ $val | quote }}

helm/minio/templates/gateway-deployment.yaml

@@ -69,8 +69,16 @@ spec:
           command:
             - "/bin/sh"
             - "-ce"
+            {{- if eq .Values.gateway.type "nas" }}
             - "/usr/bin/docker-entrypoint.sh minio gateway nas {{ $bucketRoot }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template "minio.extraArgs" . }} "
+            {{- end }}
+            {{- if eq .Values.gateway.type "gcs" }}
+            - "/usr/bin/docker-entrypoint.sh minio gateway gcs {{ .Values.gateway.gcs.projectId }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template "minio.extraArgs" . }} "
+            {{- end }}
           volumeMounts:
+            - name: minio-user
+              mountPath: "/tmp/credentials"
+              readOnly: true
             {{- if .Values.persistence.enabled }}
             - name: export
               mountPath: {{ .Values.mountPath }}
@@ -95,10 +103,34 @@ spec:
                 secretKeyRef:
                   name: {{ template "minio.secretName" . }}
                   key: rootPassword
+            {{- if eq .Values.gateway.type "gcs" }}
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/tmp/credentials/service-account-file.json"
+            {{- end }}
             {{- if .Values.metrics.serviceMonitor.public }}
             - name: MINIO_PROMETHEUS_AUTH_TYPE
               value: "public"
             {{- end}}
+            {{- if .Values.etcd.endpoints }}
+            - name: MINIO_ETCD_ENDPOINTS
+              value: {{ join "," .Values.etcd.endpoints | quote }}
+            {{- if .Values.etcd.clientCert }}
+            - name: MINIO_ETCD_CLIENT_CERT
+              value: "/tmp/credentials/etcd_client.crt"
+            {{- end }}
+            {{- if .Values.etcd.clientCertKey }}
+            - name: MINIO_ETCD_CLIENT_CERT_KEY
+              value: "/tmp/credentials/etcd_client.key"
+            {{- end }}
+            {{- if .Values.etcd.pathPrefix }}
+            - name: MINIO_ETCD_PATH_PREFIX
+              value: {{ .Values.etcd.pathPrefix }}
+            {{- end }}
+            {{- if .Values.etcd.corednsPathPrefix }}
+            - name: MINIO_ETCD_COREDNS_PATH
+              value: {{ .Values.etcd.corednsPathPrefix }}
+            {{- end }}
+            {{- end }}
             {{- range $key, $val := .Values.environment }}
             - name: {{ $key }}
               value: {{ $val | quote }}

helm/minio/templates/secrets.yaml

@@ -13,4 +13,15 @@ type: Opaque
 data:
   rootUser: {{ if .Values.rootUser }}{{ .Values.rootUser | toString | b64enc | quote }}{{ else }}{{ randAlphaNum 20 | b64enc | quote }}{{ end }}
   rootPassword: {{ if .Values.rootPassword }}{{ .Values.rootPassword | toString | b64enc | quote }}{{ else }}{{ randAlphaNum 40 | b64enc | quote }}{{ end }}
+  {{- if eq .Values.gateway.type "gcs" }}
+  {{- if .Values.gateway.gcs.serviceAccountFile }}
+  service-account-file.json: {{ .Values.gateway.gcs.serviceAccountFile | b64enc }}
+  {{- end }}
+  {{- end }}
+  {{- if .Values.etcd.clientCert }}
+  etcd_client.crt: {{ .Values.etcd.clientCert | toString | b64enc | quote }}
+  {{- end }}
+  {{- if .Values.etcd.clientCertKey }}
+  etcd_client.key: {{ .Values.etcd.clientCertKey | toString | b64enc | quote }}
+  {{- end }}
 {{- end }}

helm/minio/templates/servicemonitor.yaml

@@ -31,6 +31,9 @@ spec:
       {{- end }}
       {{- if .Values.metrics.serviceMonitor.scrapeTimeout }}
       scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.relabelConfigs }}
+{{ toYaml .Values.metrics.serviceMonitor.relabelConfigs | indent 6 }}
       {{- end }}
       {{- if not .Values.metrics.serviceMonitor.public }}
       bearerTokenSecret:

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2021-11-24T23-19-33Z
+  tag: RELEASE.2021-12-20T22-07-16Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,15 +25,11 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2021-11-16T20-37-36Z
+  tag: RELEASE.2021-12-20T23-43-34Z
   pullPolicy: IfNotPresent
 
-## minio mode, i.e. standalone or distributed or gateway (nas)
-## Distributed MinIO ref: https://docs.minio.io/docs/distributed-minio-quickstart-guide
-## NAS Gateway MinIO ref: https://docs.min.io/docs/minio-gateway-for-nas.html
-mode: distributed
-
-## NOTE: currently only "nas" gateway is supported.
+## minio mode, i.e. standalone or distributed or gateway.
+mode: distributed ## other supported values are "standalone", "gateway"
 
 ## Additional labels to include with deployment or statefulset
 additionalLabels: []
@@ -105,8 +101,11 @@ pools: 1
 
 # Deploy if 'mode == gateway' - 4 replicas.
 gateway:
-  type: "nas" # currently only "nas" is supported.
+  type: "nas" # currently only "nas,gcs" are supported.
   replicas: 4
+  gcs:
+    serviceAccountFile: "" # credential JSON file of service account key (not required if using existing secret)
+    projectId: "" # Google cloud project id required
 
 ## TLS Settings for MinIO
 tls:
@@ -183,7 +182,7 @@ ingress:
     # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0
   path: /
   hosts:
-    - chart-example.local
+    - minio-example.local
   tls: []
   #  - secretName: chart-example-tls
   #    hosts:
@@ -212,7 +211,7 @@ consoleIngress:
     # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0
   path: /
   hosts:
-    - chart-example.local
+    - console.minio-example.local
   tls: []
   #  - secretName: chart-example-tls
   #    hosts:
@@ -338,8 +337,18 @@ serviceAccount:
 metrics:
   serviceMonitor:
     enabled: false
-    additionalLabels: {}
     public: true
+    additionalLabels: {}
+    relabelConfigs: {}
     # namespace: monitoring
     # interval: 30s
     # scrapeTimeout: 10s
+
+## ETCD settings: https://github.com/minio/minio/blob/master/docs/sts/etcd.md
+## Define endpoints to enable this section.
+etcd:
+  endpoints: []
+  pathPrefix: ""
+  corednsPathPrefix: ""
+  clientCert: ""
+  clientCertKey: ""

index.yaml

@@ -1,9 +1,163 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-20T22-07-16Z
+    created: "2021-12-21T12:55:09.630617771-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.4.3.tgz
+    version: 3.4.3
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-20T22-07-16Z
+    created: "2021-12-21T12:55:09.627819129-08:00"
+    description: Multi-Cloud Object Storage
+    digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.4.2.tgz
+    version: 3.4.2
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-10T23-03-39Z
+    created: "2021-12-21T12:55:09.626348851-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.4.1.tgz
+    version: 3.4.1
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-18T04-42-33Z
+    created: "2021-12-21T12:55:09.624841158-08:00"
+    description: Multi-Cloud Object Storage
+    digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.4.0.tgz
+    version: 3.4.0
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-10T23-03-39Z
+    created: "2021-12-21T12:55:09.623402844-08:00"
+    description: Multi-Cloud Object Storage
+    digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.3.4.tgz
+    version: 3.3.4
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-10T23-03-39Z
+    created: "2021-12-21T12:55:09.621984962-08:00"
+    description: Multi-Cloud Object Storage
+    digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.3.3.tgz
+    version: 3.3.3
+  - apiVersion: v1
+    appVersion: RELEASE.2021-12-10T23-03-39Z
+    created: "2021-12-21T12:55:09.620584127-08:00"
+    description: Multi-Cloud Object Storage
+    digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.3.2.tgz
+    version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2021-12-02T12:09:05.138696778-08:00"
+    created: "2021-12-21T12:55:09.619162537-08:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -25,7 +179,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2021-12-02T12:09:05.137324882-08:00"
+    created: "2021-12-21T12:55:09.617719469-08:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -47,7 +201,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2021-12-02T12:09:05.135940325-08:00"
+    created: "2021-12-21T12:55:09.615001795-08:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -69,7 +223,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2021-12-02T12:09:05.134561831-08:00"
+    created: "2021-12-21T12:55:09.613555963-08:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -91,7 +245,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2021-12-02T12:09:05.133189479-08:00"
+    created: "2021-12-21T12:55:09.612164965-08:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -113,7 +267,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2021-12-02T12:09:05.131843522-08:00"
+    created: "2021-12-21T12:55:09.610803402-08:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -135,7 +289,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2021-12-02T12:09:05.130466001-08:00"
+    created: "2021-12-21T12:55:09.609411304-08:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -157,7 +311,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2021-12-02T12:09:05.129120348-08:00"
+    created: "2021-12-21T12:55:09.608025693-08:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -179,7 +333,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2021-12-02T12:09:05.127753234-08:00"
+    created: "2021-12-21T12:55:09.606573386-08:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -201,7 +355,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2021-12-02T12:09:05.126364514-08:00"
+    created: "2021-12-21T12:55:09.60510786-08:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -223,7 +377,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2021-12-02T12:09:05.124973115-08:00"
+    created: "2021-12-21T12:55:09.602288701-08:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -245,7 +399,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2021-12-02T12:09:05.123628402-08:00"
+    created: "2021-12-21T12:55:09.600895823-08:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -267,7 +421,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2021-12-02T12:09:05.122274696-08:00"
+    created: "2021-12-21T12:55:09.599539995-08:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -289,7 +443,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2021-12-02T12:09:05.120892474-08:00"
+    created: "2021-12-21T12:55:09.598167689-08:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -311,7 +465,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2021-12-02T12:09:05.119512001-08:00"
+    created: "2021-12-21T12:55:09.596856463-08:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -333,7 +487,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2021-12-02T12:09:05.118201395-08:00"
+    created: "2021-12-21T12:55:09.59556801-08:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -355,7 +509,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2021-12-02T12:09:05.11688914-08:00"
+    created: "2021-12-21T12:55:09.594255614-08:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -375,7 +529,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2021-12-02T12:09:05.115655396-08:00"
+    created: "2021-12-21T12:55:09.592937241-08:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -395,7 +549,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2021-12-02T12:09:05.114352488-08:00"
+    created: "2021-12-21T12:55:09.590200358-08:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -415,7 +569,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2021-12-02T12:09:05.113040515-08:00"
+    created: "2021-12-21T12:55:09.588824324-08:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -435,7 +589,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2021-12-02T12:09:05.111732781-08:00"
+    created: "2021-12-21T12:55:09.587421142-08:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -455,7 +609,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2021-12-02T12:09:05.110395434-08:00"
+    created: "2021-12-21T12:55:09.586072987-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -475,7 +629,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2021-12-02T12:09:05.109101606-08:00"
+    created: "2021-12-21T12:55:09.584735337-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -495,7 +649,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2021-12-02T12:09:05.10777829-08:00"
+    created: "2021-12-21T12:55:09.583465039-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -513,4 +667,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2021-12-02T12:09:04.547118886-08:00"
+generated: "2021-12-21T12:55:09.581829437-08:00"

internal/bucket/lifecycle/lifecycle.go

@@ -29,11 +29,10 @@ import (
 )
 
 var (
-	errLifecycleTooManyRules                = Errorf("Lifecycle configuration allows a maximum of 1000 rules")
-	errLifecycleNoRule                      = Errorf("Lifecycle configuration should have at least one rule")
-	errLifecycleDuplicateID                 = Errorf("Lifecycle configuration has rule with the same ID. Rule ID must be unique.")
-	errXMLNotWellFormed                     = Errorf("The XML you provided was not well-formed or did not validate against our published schema")
-	errLifecycleInvalidNoncurrentExpiration = Errorf("Exactly one of NoncurrentDays (positive integer) or MaxNoncurrentVersions should be specified in a NoncurrentExpiration rule.")
+	errLifecycleTooManyRules = Errorf("Lifecycle configuration allows a maximum of 1000 rules")
+	errLifecycleNoRule       = Errorf("Lifecycle configuration should have at least one rule")
+	errLifecycleDuplicateID  = Errorf("Lifecycle configuration has rule with the same ID. Rule ID must be unique.")
+	errXMLNotWellFormed      = Errorf("The XML you provided was not well-formed or did not validate against our published schema")
 )
 
 const (
@@ -141,7 +140,7 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
 		if rule.NoncurrentVersionExpiration.NoncurrentDays > 0 {
 			return true
 		}
-		if rule.NoncurrentVersionExpiration.MaxNoncurrentVersions > 0 {
+		if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 {
 			return true
 		}
 		if !rule.NoncurrentVersionTransition.IsNull() {
@@ -150,13 +149,13 @@ func (lc Lifecycle) HasActiveRules(prefix string, recursive bool) bool {
 		if rule.Expiration.IsNull() && rule.Transition.IsNull() {
 			continue
 		}
-		if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now()) {
+		if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now().UTC()) {
 			return true
 		}
 		if !rule.Expiration.IsDaysNull() {
 			return true
 		}
-		if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now()) {
+		if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now().UTC()) {
 			return true
 		}
 		if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
@@ -238,7 +237,7 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
 			rules = append(rules, rule)
 			continue
 		}
-		if rule.NoncurrentVersionExpiration.MaxNoncurrentVersions > 0 {
+		if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 {
 			rules = append(rules, rule)
 			continue
 		}
@@ -304,17 +303,24 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 				// Specifying the Days tag will automatically perform ExpiredObjectDeleteMarker cleanup
 				// once delete markers are old enough to satisfy the age criteria.
 				// https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html
-				if time.Now().After(ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))) {
+				if time.Now().UTC().After(ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days))) {
 					return DeleteVersionAction
 				}
 			}
 		}
 
 		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
+			// Skip rules with newer noncurrent versions specified.
+			// These rules are not handled at an individual version
+			// level. ComputeAction applies only to a specific
+			// version.
+			if !obj.IsLatest && rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 {
+				continue
+			}
 			if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
 				// Non current versions should be deleted if their age exceeds non current days configuration
 				// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
-				if time.Now().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays))) {
+				if time.Now().UTC().After(ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays))) {
 					return DeleteVersionAction
 				}
 			}
@@ -350,7 +356,7 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 					}
 				}
 
-				if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
+				if !obj.RestoreExpires.IsZero() && time.Now().UTC().After(obj.RestoreExpires) {
 					if obj.VersionID != "" {
 						action = DeleteRestoredVersionAction
 					} else {
@@ -358,7 +364,7 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 					}
 				}
 			}
-			if !obj.RestoreExpires.IsZero() && time.Now().After(obj.RestoreExpires) {
+			if !obj.RestoreExpires.IsZero() && time.Now().UTC().After(obj.RestoreExpires) {
 				if obj.VersionID != "" {
 					action = DeleteRestoredVersionAction
 				} else {
@@ -377,6 +383,9 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 //   e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should
 //       transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT`
 func ExpectedExpiryTime(modTime time.Time, days int) time.Time {
+	if days == 0 {
+		return modTime
+	}
 	t := modTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour)
 	return t.Truncate(24 * time.Hour)
 }
@@ -395,6 +404,11 @@ func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
 	// Iterate over all actionable rules and find the earliest
 	// expiration date and its associated rule ID.
 	for _, rule := range lc.FilterActionableRules(obj) {
+		// We don't know the index of this version and hence can't
+		// reliably compute expected expiry time.
+		if !obj.IsLatest && rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 {
+			continue
+		}
 		if !rule.NoncurrentVersionExpiration.IsDaysNull() && !obj.IsLatest && obj.VersionID != "" {
 			return rule.ID, ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays))
 		}
@@ -477,31 +491,28 @@ func (lc Lifecycle) TransitionTier(obj ObjectOpts) string {
 	return ""
 }
 
-// NoncurrentVersionsExpirationLimit returns the minimum limit on number of
+// NoncurrentVersionsExpirationLimit returns the maximum limit on number of
 // noncurrent versions across rules.
-func (lc Lifecycle) NoncurrentVersionsExpirationLimit(obj ObjectOpts) int {
+func (lc Lifecycle) NoncurrentVersionsExpirationLimit(obj ObjectOpts) (string, int, int) {
 	var lim int
+	var days int
+	var ruleID string
 	for _, rule := range lc.FilterActionableRules(obj) {
-		if rule.NoncurrentVersionExpiration.MaxNoncurrentVersions == 0 {
+		if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions == 0 {
 			continue
 		}
-		if lim == 0 || lim > rule.NoncurrentVersionExpiration.MaxNoncurrentVersions {
-			lim = rule.NoncurrentVersionExpiration.MaxNoncurrentVersions
+		// Pick the highest number of NewerNoncurrentVersions value
+		// among overlapping rules.
+		if lim == 0 || lim < rule.NoncurrentVersionExpiration.NewerNoncurrentVersions {
+			lim = rule.NoncurrentVersionExpiration.NewerNoncurrentVersions
+		}
+		// Pick the earliest applicable NoncurrentDays among overlapping
+		// rules. Note: ruleID is that of the rule which determines the
+		// time of expiry.
+		if ndays := int(rule.NoncurrentVersionExpiration.NoncurrentDays); days == 0 || days > ndays {
+			days = ndays
+			ruleID = rule.ID
 		}
 	}
-	return lim
-}
-
-// HasMaxNoncurrentVersions returns true if there exists a rule with
-// MaxNoncurrentVersions limit set.
-func (lc Lifecycle) HasMaxNoncurrentVersions() bool {
-	for _, rule := range lc.Rules {
-		if rule.Status == Disabled {
-			continue
-		}
-		if rule.NoncurrentVersionExpiration.MaxNoncurrentVersions > 0 {
-			return true
-		}
-	}
-	return false
+	return ruleID, days, lim
 }

internal/bucket/lifecycle/lifecycle_test.go

@@ -114,7 +114,7 @@ func TestParseAndValidateLifecycleConfig(t *testing.T) {
 		},
 		// Lifecycle with max noncurrent versions
 		{
-			inputConfig:           `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID>><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><MaxNoncurrentVersions>5</MaxNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>`,
+			inputConfig:           `<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID>><Status>Enabled</Status><Filter></Filter><NoncurrentVersionExpiration><NewerNoncurrentVersions>5</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>`,
 			expectedParsingErr:    nil,
 			expectedValidationErr: nil,
 		},
@@ -414,6 +414,24 @@ func TestComputeActions(t *testing.T) {
 			objectSuccessorModTime: time.Now().Add(-1 * time.Nanosecond).UTC(),
 			versionID:              uuid.New().String(),
 		},
+		// Lifecycle rules with NewerNoncurrentVersions specified must return NoneAction.
+		{
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><NoncurrentVersionExpiration><NewerNoncurrentVersions>5</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>`,
+			objectName:     "foodir/fooobject",
+			versionID:      uuid.NewString(),
+			objectModTime:  time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			expectedAction: NoneAction,
+		},
+		// Disabled rules with NewerNoncurrentVersions shouldn't affect outcome.
+		{
+			inputConfig:            `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><NoncurrentVersionExpiration><NoncurrentDays>5</NoncurrentDays></NoncurrentVersionExpiration></Rule><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Disabled</Status><NoncurrentVersionExpiration><NewerNoncurrentVersions>5</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>`,
+			objectName:             "foodir/fooobject",
+			versionID:              uuid.NewString(),
+			objectModTime:          time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			objectSuccessorModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago
+			isNoncurrent:           true,
+			expectedAction:         DeleteVersionAction,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -636,14 +654,55 @@ func TestNoncurrentVersionsLimit(t *testing.T) {
 			ID:     strconv.Itoa(i),
 			Status: "Enabled",
 			NoncurrentVersionExpiration: NoncurrentVersionExpiration{
-				MaxNoncurrentVersions: i,
+				NewerNoncurrentVersions: i,
+				NoncurrentDays:          ExpirationDays(i),
 			},
 		})
 	}
 	lc := Lifecycle{
 		Rules: rules,
 	}
-	if lim := lc.NoncurrentVersionsExpirationLimit(ObjectOpts{Name: "obj"}); lim != 1 {
-		t.Fatalf("Expected max noncurrent versions limit to be 1 but got %d", lim)
+	if ruleID, days, lim := lc.NoncurrentVersionsExpirationLimit(ObjectOpts{Name: "obj"}); ruleID != "1" || days != 1 || lim != 10 {
+		t.Fatalf("Expected (ruleID, days, lim) to be (\"1\", 1, 10) but got (%s, %d, %d)", ruleID, days, lim)
+	}
+}
+
+func TestMaxNoncurrentBackwardCompat(t *testing.T) {
+	testCases := []struct {
+		xml      string
+		expected NoncurrentVersionExpiration
+	}{
+		{
+			xml: `<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays><NewerNoncurrentVersions>3</NewerNoncurrentVersions></NoncurrentVersionExpiration>`,
+			expected: NoncurrentVersionExpiration{
+				XMLName: xml.Name{
+					Local: "NoncurrentVersionExpiration",
+				},
+				NoncurrentDays:          1,
+				NewerNoncurrentVersions: 3,
+				set:                     true,
+			},
+		},
+		{
+			xml: `<NoncurrentVersionExpiration><NoncurrentDays>2</NoncurrentDays><MaxNoncurrentVersions>4</MaxNoncurrentVersions></NoncurrentVersionExpiration>`,
+			expected: NoncurrentVersionExpiration{
+				XMLName: xml.Name{
+					Local: "NoncurrentVersionExpiration",
+				},
+				NoncurrentDays:          2,
+				NewerNoncurrentVersions: 4,
+				set:                     true,
+			},
+		},
+	}
+	for i, tc := range testCases {
+		var got NoncurrentVersionExpiration
+		dec := xml.NewDecoder(strings.NewReader(tc.xml))
+		if err := dec.Decode(&got); err != nil || got != tc.expected {
+			if err != nil {
+				t.Fatalf("%d: Failed to unmarshal xml %v", i+1, err)
+			}
+			t.Fatalf("%d: Expected %v but got %v", i+1, tc.expected, got)
+		}
 	}
 }

internal/bucket/lifecycle/noncurrentversion.go

@@ -24,10 +24,10 @@ import (
 
 // NoncurrentVersionExpiration - an action for lifecycle configuration rule.
 type NoncurrentVersionExpiration struct {
-	XMLName               xml.Name       `xml:"NoncurrentVersionExpiration"`
-	NoncurrentDays        ExpirationDays `xml:"NoncurrentDays,omitempty"`
-	MaxNoncurrentVersions int            `xml:"MaxNoncurrentVersions,omitempty"`
-	set                   bool
+	XMLName                 xml.Name       `xml:"NoncurrentVersionExpiration"`
+	NoncurrentDays          ExpirationDays `xml:"NoncurrentDays,omitempty"`
+	NewerNoncurrentVersions int            `xml:"NewerNoncurrentVersions,omitempty"`
+	set                     bool
 }
 
 // MarshalXML if non-current days not set to non zero value
@@ -41,20 +41,35 @@ func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartE
 
 // UnmarshalXML decodes NoncurrentVersionExpiration
 func (n *NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
-	type noncurrentVersionExpirationWrapper NoncurrentVersionExpiration
-	var val noncurrentVersionExpirationWrapper
+	// To handle xml with MaxNoncurrentVersions from older MinIO releases.
+	// note: only one of MaxNoncurrentVersions or NewerNoncurrentVersions would be present.
+	type noncurrentExpiration struct {
+		XMLName                 xml.Name       `xml:"NoncurrentVersionExpiration"`
+		NoncurrentDays          ExpirationDays `xml:"NoncurrentDays,omitempty"`
+		NewerNoncurrentVersions int            `xml:"NewerNoncurrentVersions,omitempty"`
+		MaxNoncurrentVersions   int            `xml:"MaxNoncurrentVersions,omitempty"`
+	}
+
+	var val noncurrentExpiration
 	err := d.DecodeElement(&val, &startElement)
 	if err != nil {
 		return err
 	}
-	*n = NoncurrentVersionExpiration(val)
+	if val.MaxNoncurrentVersions > 0 {
+		val.NewerNoncurrentVersions = val.MaxNoncurrentVersions
+	}
+	*n = NoncurrentVersionExpiration{
+		XMLName:                 val.XMLName,
+		NoncurrentDays:          val.NoncurrentDays,
+		NewerNoncurrentVersions: val.NewerNoncurrentVersions,
+	}
 	n.set = true
 	return nil
 }
 
 // IsNull returns if both NoncurrentDays and NoncurrentVersions are empty
 func (n NoncurrentVersionExpiration) IsNull() bool {
-	return n.IsDaysNull() && n.MaxNoncurrentVersions == 0
+	return n.IsDaysNull() && n.NewerNoncurrentVersions == 0
 }
 
 // IsDaysNull returns true if days field is null
@@ -69,16 +84,13 @@ func (n NoncurrentVersionExpiration) Validate() error {
 	}
 	val := int(n.NoncurrentDays)
 	switch {
-	case val == 0 && n.MaxNoncurrentVersions == 0:
+	case val == 0 && n.NewerNoncurrentVersions == 0:
 		// both fields can't be zero
 		return errXMLNotWellFormed
 
-	case val > 0 && n.MaxNoncurrentVersions > 0:
-		// both tags can't be non-zero simultaneously
-		return errLifecycleInvalidNoncurrentExpiration
-
-	case val < 0, n.MaxNoncurrentVersions < 0:
+	case val < 0, n.NewerNoncurrentVersions < 0:
 		// negative values are not supported
+		return errXMLNotWellFormed
 	}
 	return nil
 }

internal/bucket/lifecycle/noncurrentversion_test.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package lifecycle
+
+import "testing"
+
+func Test_NoncurrentVersionsExpiration_Validation(t *testing.T) {
+	testcases := []struct {
+		n   NoncurrentVersionExpiration
+		err error
+	}{
+		{
+			n: NoncurrentVersionExpiration{
+				NoncurrentDays:          0,
+				NewerNoncurrentVersions: 0,
+				set:                     true,
+			},
+			err: errXMLNotWellFormed,
+		},
+		{
+			n: NoncurrentVersionExpiration{
+				NoncurrentDays:          90,
+				NewerNoncurrentVersions: 0,
+				set:                     true,
+			},
+			err: nil,
+		},
+		{
+			n: NoncurrentVersionExpiration{
+				NoncurrentDays:          90,
+				NewerNoncurrentVersions: 2,
+				set:                     true,
+			},
+			err: nil,
+		},
+		{
+			n: NoncurrentVersionExpiration{
+				NoncurrentDays: -1,
+				set:            true,
+			},
+			err: errXMLNotWellFormed,
+		},
+		{
+			n: NoncurrentVersionExpiration{
+				NoncurrentDays:          90,
+				NewerNoncurrentVersions: -2,
+				set:                     true,
+			},
+			err: errXMLNotWellFormed,
+		},
+		// MinIO extension: supports zero NoncurrentDays when NewerNoncurrentVersions > 0
+		{
+			n: NoncurrentVersionExpiration{
+				NoncurrentDays:          0,
+				NewerNoncurrentVersions: 5,
+				set:                     true,
+			},
+			err: nil,
+		},
+	}
+
+	for i, tc := range testcases {
+		if got := tc.n.Validate(); got != tc.err {
+			t.Fatalf("%d: expected %v but got %v", i+1, tc.err, got)
+		}
+	}
+}