Ensure lock/versioning permissions when creating a bucket (#15432)

Currently, the code doesn't check if the user creating a bucket with
locking feature has bucket locking and versioning permissions enabled,
adding it in accordance with S3 spec.

https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html

Object Lock - If ObjectLockEnabledForBucket is set to true in your CreateBucket request,
s3:PutBucketObjectLockConfiguration and s3:PutBucketVersioning permissions are required.

.github/workflows/depsreview.yaml

@@ -0,0 +1,14 @@
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v1

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
@@ -49,3 +49,4 @@ jobs:
           make verify-healing
           make verify-healing-inconsistent-versions
           make verify-healing-with-root-disks
+          make verify-healing-with-rewrite

.github/workflows/go-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]

.github/workflows/replication.yaml

@@ -21,7 +21,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
 
     steps:

.golangci.yml

@@ -14,13 +14,9 @@ linters:
     - govet
     - revive
     - ineffassign
-    - gosimple
     - deadcode
-    - structcheck
     - gomodguard
     - gofmt
-    - unused
-    - structcheck
     - unconvert
     - varcheck
     - gocritic

Makefile

@@ -15,7 +15,7 @@ checks: ## check dependencies
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
 help: ## print this help
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}'
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
@@ -90,11 +90,16 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
-verify-healing-with-root-disks:
-	@echo "Verify healing with root disks"
+verify-healing-with-root-disks: ## verify healing root disks
+	@echo "Verify healing with root drives"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
 
+verify-healing-with-rewrite: ## verify healing to rewrite old xl.meta -> new xl.meta
+	@echo "Verify healing with rewrite"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/rewrite-old-new.sh)
+
 verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

README.md

@@ -125,7 +125,7 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go install github.com/minio/minio@latest

buildscripts/heal-manual.go

@@ -0,0 +1,87 @@
+//go:build ignore
+// +build ignore
+
+//
+// MinIO Object Storage (c) 2022 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New(os.Args[1], os.Args[2], os.Args[3], false)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	opts := madmin.HealOpts{
+		Recursive: true,                  // recursively heal all objects at 'prefix'
+		Remove:    true,                  // remove content that has lost quorum and not recoverable
+		Recreate:  true,                  // rewrite all old non-inlined xl.meta to new xl.meta
+		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
+	}
+
+	start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	fmt.Println("Healstart sequence ===")
+	enc := json.NewEncoder(os.Stdout)
+	if err = enc.Encode(&start); err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Println()
+	for {
+		_, status, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, start.ClientToken, false, false)
+		if status.Summary == "finished" {
+			fmt.Println("Healstatus on items ===")
+			for _, item := range status.Items {
+				if err = enc.Encode(&item); err != nil {
+					log.Fatalln(err)
+				}
+			}
+			break
+		}
+		if status.Summary == "stopped" {
+			fmt.Println("Healstatus on items ===")
+			fmt.Println("Heal failed with", status.FailureDetail)
+			break
+		}
+
+		for _, item := range status.Items {
+			if err = enc.Encode(&item); err != nil {
+				log.Fatalln(err)
+			}
+		}
+
+		time.Sleep(time.Second)
+	}
+}

buildscripts/resolve-right-versions.sh

@@ -39,7 +39,7 @@ function start_minio_5drive() {
     "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
     pid=$!
     disown $pid
-    sleep 30
+    sleep 5
 
     if ! ps -p ${pid} 1>&2 >/dev/null; then
 	echo "server1 log:"

buildscripts/rewrite-old-new.sh

@@ -0,0 +1,151 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
+	chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
+    fi
+}
+
+function verify_rewrite() {
+    start_port=$1
+
+    export MINIO_ACCESS_KEY=minio
+    export MINIO_SECRET_KEY=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export MINIO_CI_CD=1
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 10
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    kill ${pid}
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 10
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -versions \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
+    sleep 1
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -versions \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    kill ${pid}
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    verify_rewrite ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

cmd/acl-handlers.go

@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/admin-bucket-handlers.go

@@ -69,7 +69,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -131,7 +131,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -173,7 +173,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -301,7 +301,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	}
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -340,7 +340,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -374,7 +374,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	bucket := pathClean(r.Form.Get("bucket"))
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -390,13 +390,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	)
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 		buckets = append(buckets, BucketInfo{Name: bucket})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx)
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -619,6 +619,39 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	}
 }
 
+type importMetaReport struct {
+	madmin.BucketMetaImportErrs
+}
+
+func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
+	st := i.Buckets[bucket]
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	switch fname {
+	case bucketPolicyConfig:
+		st.Policy = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketNotificationConfig:
+		st.Notification = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketLifecycleConfig:
+		st.Lifecycle = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketSSEConfig:
+		st.SSEConfig = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketTaggingConfig:
+		st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketQuotaConfigFile:
+		st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case objectLockConfig:
+		st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketVersioningConfig:
+		st.Versioning = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	default:
+		st.Err = errMsg
+	}
+	i.Buckets[bucket] = st
+}
+
 // ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config
 // There are some caveats regarding the following:
 // 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here.
@@ -628,9 +661,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	ctx := newContext(r, w, "ImportBucketMetadata")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	bucket := pathClean(r.Form.Get("bucket"))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -651,47 +683,46 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		return
 	}
 	bucketMap := make(map[string]struct{}, 1)
-
+	rpt := importMetaReport{
+		madmin.BucketMetaImportErrs{
+			Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
+		},
+	}
 	// import object lock config if any - order of import matters here.
 	for _, file := range zr.File {
 		slc := strings.Split(file.Name, slashSeparator)
 		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-			return
-		}
-		b, fileName := slc[0], slc[1]
-		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
-			bucket = b
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
 		}
+		bucket, fileName := slc[0], slc[1]
 		switch fileName {
 		case objectLockConfig:
 			reader, err := file.Open()
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			config, err := objectlock.ParseObjectLockConfig(reader)
 			if err != nil {
-				apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
-				apiErr.Description = err.Error()
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			configData, err := xml.Marshal(config)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				opts := BucketOptions{
+				opts := MakeBucketOptions{
 					LockEnabled: config.ObjectLockEnabled == "Enabled",
 				}
 				err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
 				if err != nil {
 					if _, ok := err.(BucketExists); !ok {
-						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-						return
+						rpt.SetStatus(bucket, fileName, err)
+						continue
 					}
 				}
 				bucketMap[bucket] = struct{}{}
@@ -699,15 +730,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Deny object locking configuration settings on existing buckets without object lock enabled.
 			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 
 			// Call site replication hook.
 			//
@@ -720,8 +752,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				ObjectLockConfig: &cfgStr,
 				UpdatedAt:        updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 		}
 	}
@@ -730,162 +762,145 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	for _, file := range zr.File {
 		slc := strings.Split(file.Name, slashSeparator)
 		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-			return
-		}
-		b, fileName := slc[0], slc[1]
-		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
-			bucket = b
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
 		}
+		bucket, fileName := slc[0], slc[1]
 		switch fileName {
 		case bucketVersioningConfig:
 			reader, err := file.Open()
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize))
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
-				if err != nil {
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
 					if _, ok := err.(BucketExists); !ok {
-						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-						return
+						rpt.SetStatus(bucket, fileName, err)
+						continue
 					}
 				}
 				bucketMap[bucket] = struct{}{}
 			}
 
 			if globalSiteReplicationSys.isEnabled() && v.Suspended() {
-				writeErrorResponse(ctx, w, APIError{
-					Code:           "InvalidBucketState",
-					Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
-					HTTPStatusCode: http.StatusConflict,
-				}, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
+				continue
 			}
 
 			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
-				writeErrorResponse(ctx, w, APIError{
-					Code:           "InvalidBucketState",
-					Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
-					HTTPStatusCode: http.StatusConflict,
-				}, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
+				continue
 			}
 			if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
-				writeErrorResponse(ctx, w, APIError{
-					Code:           "InvalidBucketState",
-					Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
-					HTTPStatusCode: http.StatusConflict,
-				}, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
+				continue
 			}
 
 			configData, err := xml.Marshal(v)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 		}
 	}
 
 	for _, file := range zr.File {
 		reader, err := file.Open()
 		if err != nil {
-			writeErrorResponse(ctx, w, importError(ctx, err, file.Name, ""), r.URL)
-			return
+			rpt.SetStatus(file.Name, "", err)
+			continue
 		}
 		sz := file.FileInfo().Size()
 		slc := strings.Split(file.Name, slashSeparator)
 		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-			return
-		}
-		b, fileName := slc[0], slc[1]
-		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
-			bucket = b
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
 		}
+		bucket, fileName := slc[0], slc[1]
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				if _, ok := err.(BucketExists); !ok {
-					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-					return
+					rpt.SetStatus(bucket, "", err)
+					continue
 				}
 			}
 			bucketMap[bucket] = struct{}{}
 		}
+		if _, ok := bucketMap[bucket]; !ok {
+			continue
+		}
 		switch fileName {
 		case bucketNotificationConfig:
 			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalNotificationSys.targetList)
 			if err != nil {
-				apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
-				if event.IsEventError(err) {
-					apiErr = importError(ctx, err, file.Name, bucket)
-				}
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			configData, err := xml.Marshal(config)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			rulesMap := config.ToRulesMap()
 			globalNotificationSys.AddRulesMap(bucket, rulesMap)
+			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
 			if sz > maxBucketPolicySize {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
+				continue
 			}
 
 			bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(reader, sz))
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Version in policy must not be empty
 			if bucketPolicy.Version == "" {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrMalformedPolicy.String()))
+				continue
 			}
 
 			configData, err := json.Marshal(bucketPolicy)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 			// Call site replication hook.
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 				Type:      madmin.SRBucketMetaTypePolicy,
@@ -893,56 +908,51 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Policy:    bucketPolicyBytes,
 				UpdatedAt: updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
-
 		case bucketLifecycleConfig:
 			bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Validate the received bucket policy document
 			if err = bucketLifecycle.Validate(); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Validate the transition storage ARNs
 			if err = validateTransitionTier(bucketLifecycle); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			configData, err := xml.Marshal(bucketLifecycle)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 		case bucketSSEConfig:
 			// Parse bucket encryption xml
 			encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize))
 			if err != nil {
-				apiErr := APIError{
-					Code:           "MalformedXML",
-					Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err),
-					HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
-				}
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			// Return error if KMS is not initialized
 			if GlobalKMS == nil {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s", errorCodes[ErrKMSNotConfigured].Description))
+				continue
 			}
 			kmsKey := encConfig.KeyID()
 			if kmsKey != "" {
@@ -950,26 +960,27 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
 				if err != nil {
 					if errors.Is(err, kes.ErrKeyNotFound) {
-						writeErrorResponse(ctx, w, importError(ctx, errKMSKeyNotFound, file.Name, bucket), r.URL)
-						return
+						rpt.SetStatus(bucket, fileName, errKMSKeyNotFound)
+						continue
 					}
-					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-					return
+					rpt.SetStatus(bucket, fileName, err)
+					continue
 				}
 			}
 
 			configData, err := xml.Marshal(encConfig)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Store the bucket encryption configuration in the object layer
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 
 			// Call site replication hook.
 			//
@@ -982,29 +993,30 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				SSEConfig: &cfgStr,
 				UpdatedAt: updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 		case bucketTaggingConfig:
 			tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
 			if err != nil {
-				apiErr := errorCodes.ToAPIErrWithErr(ErrMalformedXML, fmt.Errorf("error importing %s with %w", file.Name, err))
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			configData, err := xml.Marshal(tags)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
+
 			// Call site replication hook.
 			//
 			// We encode the xml bytes as base64 to ensure there are no encoding
@@ -1016,32 +1028,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Tags:      &cfgStr,
 				UpdatedAt: updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 		case bucketQuotaConfigFile:
 			data, err := ioutil.ReadAll(reader)
 			if err != nil {
-				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			quotaConfig, err := parseBucketQuota(bucket, data)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			if quotaConfig.Type == "fifo" {
-				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 
 			bucketMeta := madmin.SRBucketMeta{
 				Type:      madmin.SRBucketMetaTypeQuotaConfig,
@@ -1055,11 +1068,19 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Call site replication hook.
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 		}
 	}
+
+	rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, rptData)
 }
 
 // ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
@@ -1089,7 +1110,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-config-kv.go

@@ -189,15 +189,20 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	cfg := globalServerConfig.Clone()
 	vars := mux.Vars(r)
-	buf := &bytes.Buffer{}
-	cw := config.NewConfigWriteTo(cfg, vars["key"])
-	if _, err := cw.WriteTo(buf); err != nil {
+	subSys := vars["key"]
+	subSysConfigs, err := cfg.GetSubsysInfo(subSys)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	var s strings.Builder
+	for _, subSysConfig := range subSysConfigs {
+		subSysConfig.AddString(&s, false)
+	}
+
 	password := cred.SecretKey
-	econfigData, err := madmin.EncryptData(password, buf.Bytes())
+	econfigData, err := madmin.EncryptData(password, []byte(s.String()))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -427,42 +432,30 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		count += len(cfg[hkv.Key])
 	}
 	for _, hkv := range hkvs {
-		v := cfg[hkv.Key]
-		for target, kv := range v {
-			off := kv.Get(config.Enable) == config.EnableOff
+		// We ignore the error below, as we cannot get one.
+		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key)
+
+		for _, item := range cfgSubsysItems {
+			off := item.Params.Get(config.Enable) == config.EnableOff
 			switch hkv.Key {
 			case config.EtcdSubSys:
-				off = !etcd.Enabled(kv)
+				off = !etcd.Enabled(item.Params)
 			case config.CacheSubSys:
-				off = !cache.Enabled(kv)
+				off = !cache.Enabled(item.Params)
 			case config.StorageClassSubSys:
-				off = !storageclass.Enabled(kv)
+				off = !storageclass.Enabled(item.Params)
 			case config.PolicyPluginSubSys:
-				off = !polplugin.Enabled(kv)
+				off = !polplugin.Enabled(item.Params)
 			case config.IdentityOpenIDSubSys:
-				off = !openid.Enabled(kv)
+				off = !openid.Enabled(item.Params)
 			case config.IdentityLDAPSubSys:
-				off = !xldap.Enabled(kv)
+				off = !xldap.Enabled(item.Params)
 			case config.IdentityTLSSubSys:
 				off = !globalSTSTLSConfig.Enabled
 			case config.IdentityPluginSubSys:
-				off = !idplugin.Enabled(kv)
-			}
-			if off {
-				s.WriteString(config.KvComment)
-				s.WriteString(config.KvSpaceSeparator)
-			}
-			s.WriteString(hkv.Key)
-			if target != config.Default {
-				s.WriteString(config.SubSystemSeparator)
-				s.WriteString(target)
-			}
-			s.WriteString(config.KvSpaceSeparator)
-			s.WriteString(kv.String())
-			count--
-			if count > 0 {
-				s.WriteString(config.KvNewline)
+				off = !idplugin.Enabled(item.Params)
 			}
+			item.AddString(&s, off)
 		}
 	}
 

cmd/admin-handlers-site-replication.go

@@ -24,6 +24,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -116,19 +118,38 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
 		_, isForceCreate := r.Form["forceCreate"]
-		opts := BucketOptions{
+		createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
+		createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
+		if cerr != nil {
+			createdAt = timeSentinel
+		}
+
+		opts := MakeBucketOptions{
 			Location:          r.Form.Get("location"),
 			LockEnabled:       isLockEnabled,
 			VersioningEnabled: isVersioningEnabled,
 			ForceCreate:       isForceCreate,
+			CreatedAt:         createdAt,
 		}
 		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 	case madmin.ConfigureReplBktOp:
 		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
 	case madmin.DeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      false,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
 	case madmin.ForceDeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      true,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
+	case madmin.PurgeDeletedBucketOp:
+		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -436,6 +457,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	opts.Users = q.Get("users") == "true"
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
+	opts.ShowDeleted = q.Get("showDeleted") == "true"
 	return
 }
 

cmd/admin-handlers-users.go

@@ -1177,7 +1177,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx)
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1189,17 +1189,32 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		// For derived credentials, check the parent user's permissions.
 		accountName = cred.ParentUser
 	}
-	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 
-	buf, err := json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+	var buf []byte
+	if accountName == globalActiveCred.AccessKey {
+		for _, policy := range iampolicy.DefaultPolicies {
+			if policy.Name == "consoleAdmin" {
+				buf, err = json.MarshalIndent(policy.Definition, "", " ")
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				break
+			}
+		}
+	} else {
+		policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		buf, err = json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	acctInfo := madmin.AccountInfo{

cmd/admin-handlers.go

@@ -71,16 +71,6 @@ const (
 	mgmtForceStop   = "forceStop"
 )
 
-func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) {
-	if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil {
-		return us, err
-	}
-
-	us.CurrentVersion = Version
-	us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
-	return us, nil
-}
-
 // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
@@ -152,7 +142,16 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
+	// Download Binary Once
+	reader, err := downloadBinary(u, mode)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Push binary to other servers
+	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
 		if nerr.Err != nil {
 			err := AdminError{
 				Code:       AdminUpdateApplyFailure,
@@ -166,13 +165,39 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
+	err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	for _, nerr := range globalNotificationSys.CommitBinary(ctx) {
+		if nerr.Err != nil {
+			err := AdminError{
+				Code:       AdminUpdateApplyFailure,
+				Message:    nerr.Err.Error(),
+				StatusCode: http.StatusInternalServerError,
+			}
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	err = commitBinary()
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	updateStatus := madmin.ServerUpdateStatus{
+		CurrentVersion: Version,
+		UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+	}
+
 	// Marshal API response
 	jsonBytes, err := json.Marshal(updateStatus)
 	if err != nil {
@@ -1190,16 +1215,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		concurrent = 32
 	}
 
-	if runtime.GOMAXPROCS(0) < concurrent {
-		concurrent = runtime.GOMAXPROCS(0)
-	}
-
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
 		duration = time.Second * 10
 	}
 
-	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
+
+	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
 			Code:       "XMinioSpeedtestInsufficientCapacity",
@@ -1272,7 +1295,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 }
 
 func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
-	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
+	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
@@ -1286,11 +1309,11 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 	objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 		Force:      true,
 		NoRecreate: true,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 	})
 }
 
-func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
-	storageInfo, _ := objectAPI.StorageInfo(ctx)
+func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, concurrent int, size int, autotune bool) (bool, bool, string) {
 	capacityNeeded := uint64(concurrent * size)
 	capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
 
@@ -2199,15 +2222,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	getAndWriteObjPerfInfo := func() {
 		if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
 			concurrent := 32
-			if runtime.GOMAXPROCS(0) < concurrent {
-				concurrent = runtime.GOMAXPROCS(0)
-			}
+
+			storageInfo, _ := objectAPI.StorageInfo(ctx)
 
 			size := 64 * humanize.MiByte
 			autotune := true
 
-			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
-
+			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 			if !sufficientCapacity {
 				healthInfo.Perf.Error = capacityErrMsg
 				partialWrite(healthInfo)
@@ -2636,6 +2657,100 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	return nil
 }
 
+type clusterInfo struct {
+	DeploymentID string `json:"deployment_id"`
+	ClusterName  string `json:"cluster_name"`
+	UsedCapacity uint64 `json:"used_capacity"`
+	Info         struct {
+		MinioVersion    string `json:"minio_version"`
+		PoolsCount      int    `json:"pools_count"`
+		ServersCount    int    `json:"servers_count"`
+		DrivesCount     int    `json:"drives_count"`
+		BucketsCount    uint64 `json:"buckets_count"`
+		ObjectsCount    uint64 `json:"objects_count"`
+		TotalDriveSpace uint64 `json:"total_drive_space"`
+		UsedDriveSpace  uint64 `json:"used_drive_space"`
+	} `json:"info"`
+}
+
+func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
+	// Send profiling data to zip as file
+	header, zerr := zip.FileInfoHeader(dummyFileInfo{
+		name:    name,
+		size:    int64(len(data)),
+		mode:    0o600,
+		modTime: UTCNow(),
+		isDir:   false,
+		sys:     nil,
+	})
+	if zerr != nil {
+		return zerr
+	}
+	header.Method = zip.Deflate
+	zwriter, zerr := zipWriter.CreateHeader(header)
+	if zerr != nil {
+		return zerr
+	}
+	_, err := io.Copy(zwriter, bytes.NewReader(data))
+	return err
+}
+
+// appendClusterMetaInfoToZip gets information of the current cluster and embedded
+// it in the passed zipwriter, This is not a critical function and it is allowed
+// to fail with a ten seconds timeout.
+func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return
+	}
+
+	// Add a ten seconds timeout because getting profiling data
+	// is critical for debugging, in contrary to getting cluster info
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	resultCh := make(chan clusterInfo)
+	go func() {
+		ci := clusterInfo{}
+		ci.Info.PoolsCount = len(globalEndpoints)
+		ci.Info.ServersCount = len(globalEndpoints.Hostnames())
+		ci.Info.MinioVersion = Version
+
+		si, _ := objectAPI.StorageInfo(ctx)
+
+		ci.Info.DrivesCount = len(si.Disks)
+		for _, disk := range si.Disks {
+			ci.Info.TotalDriveSpace += disk.TotalSpace
+			ci.Info.UsedDriveSpace += disk.UsedSpace
+		}
+
+		dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
+
+		ci.UsedCapacity = dataUsageInfo.ObjectsTotalSize
+		ci.Info.BucketsCount = dataUsageInfo.BucketsCount
+		ci.Info.ObjectsCount = dataUsageInfo.ObjectsTotalCount
+
+		ci.DeploymentID = globalDeploymentID
+		ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.ServersCount, ci.Info.DrivesCount, ci.Info.MinioVersion)
+		resultCh <- ci
+	}()
+
+	select {
+	case <-ctx.Done():
+		return
+	case ci := <-resultCh:
+		out, err := json.MarshalIndent(ci, "", "   ")
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return
+		}
+		err = embedFileInZip(zipWriter, "cluster.info", out)
+		if err != nil {
+			logger.LogIf(ctx, err)
+		}
+	}
+}
+
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
 	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
@@ -2655,7 +2770,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	o, ok := newObjectLayerFn().(getRawDataer)
+	objLayer := newObjectLayerFn()
+	o, ok := objLayer.(getRawDataer)
 	if !ok {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
@@ -2778,6 +2894,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}); err != nil {
 		logger.LogIf(ctx, err)
 	}
+
+	appendClusterMetaInfoToZip(ctx, zipWriter)
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-heal-ops.go

@@ -822,7 +822,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	buckets, err := objAPI.ListBuckets(h.ctx)
+	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}

cmd/api-errors.go

@@ -1158,7 +1158,7 @@ var errorCodes = errorCodeMap{
 	// MinIO extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
-		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
+		Description:    "Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
 	ErrRequestBodyParse: {

cmd/api-headers.go

@@ -199,5 +199,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 	}
 
+	if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
+		if i := strings.LastIndexByte(v, '/'); i >= 0 {
+			v = v[i+1:]
+		}
+		w.Header()[xhttp.MinIOCompressed] = []string{v}
+	}
+
 	return nil
 }

cmd/background-newdisks-heal-ops.go

@@ -87,7 +87,7 @@ type healingTracker struct {
 // The disk ID will be validated against the loaded one.
 func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) {
 	if disk == nil {
-		return nil, errors.New("loadHealingTracker: nil disk given")
+		return nil, errors.New("loadHealingTracker: nil drive given")
 	}
 	diskID, err := disk.GetDiskID()
 	if err != nil {
@@ -104,7 +104,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 		return nil, err
 	}
 	if h.ID != diskID && h.ID != "" {
-		return nil, fmt.Errorf("loadHealingTracker: disk id mismatch expected %s, got %s", h.ID, diskID)
+		return nil, fmt.Errorf("loadHealingTracker: drive id mismatch expected %s, got %s", h.ID, diskID)
 	}
 	h.disk = disk
 	h.ID = diskID
@@ -129,7 +129,7 @@ func newHealingTracker(disk StorageAPI) *healingTracker {
 // If the tracker has been deleted an error is returned.
 func (h *healingTracker) update(ctx context.Context) error {
 	if h.disk.Healing() == nil {
-		return fmt.Errorf("healingTracker: disk %q is not marked as healing", h.ID)
+		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
 	}
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
@@ -310,7 +310,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	// Prevent parallel erasure set healing
-	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-disk-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
+	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
 	if err != nil {
 		return err
@@ -318,8 +318,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx.Cancel)
 
-	buckets, _ := z.ListBuckets(ctx)
-
+	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to heal all bucket metadata configuration.
 	buckets = append(buckets, BucketInfo{
@@ -338,14 +337,14 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	})
 
 	if serverDebugLog {
-		logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
+		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
 	}
 
 	// Load healing tracker in this disk
 	tracker, err := loadHealingTracker(ctx, disk)
 	if err != nil {
 		// So someone changed the drives underneath, healing tracker missing.
-		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
+		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
 			disk, humanize.Ordinal(poolIdx+1), err))
 		tracker = newHealingTracker(disk)
 	}
@@ -370,9 +369,9 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	} else {
-		logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	}
 
 	if serverDebugLog {

cmd/benchmark-utils_test.go

@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bitrot-streaming.go

@@ -180,7 +180,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s  -> %s/%s - content hash does not match - expected %s, got %s",
 			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
 		return 0, errFileCorrupt
 	}

cmd/bitrot-whole.go

@@ -38,12 +38,12 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -72,12 +72,12 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
+			logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot_test.go

@@ -20,17 +20,11 @@ package cmd
 import (
 	"context"
 	"io"
-	"io/ioutil"
-	"os"
 	"testing"
 )
 
 func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
-	tmpDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	volume := "testvol"
 	filePath := "testfile"
@@ -60,7 +54,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	writer.(io.Closer).Close()
+	if bw, ok := writer.(io.Closer); ok {
+		bw.Close()
+	}
 
 	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
@@ -76,6 +72,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if _, err = reader.ReadAt(b[:5], 30); err != nil {
 		t.Fatal(err)
 	}
+	if br, ok := reader.(io.Closer); ok {
+		br.Close()
+	}
 }
 
 func TestAllBitrotAlgorithms(t *testing.T) {

cmd/bucket-encryption-handlers.go

@@ -65,7 +65,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -155,7 +155,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -198,7 +198,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -201,7 +201,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -332,7 +332,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	} else {
 		// Invoke the list buckets.
 		var err error
-		bucketsInfo, err = listBuckets(ctx)
+		bucketsInfo, err = listBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -447,7 +447,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
 
 	// Before proceeding validate if bucket exists.
-	_, err := objectAPI.GetBucketInfo(ctx, bucket)
+	_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -727,11 +727,30 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
+	cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction, bucket, "")
+	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 
+	if objectLockEnabled {
+		// Creating a bucket with locking requires the user having more permissions
+		for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
+				Action:          action,
+				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				BucketName:      bucket,
+				IsOwner:         owner,
+				Claims:          cred.Claims,
+			}) {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+				return
+			}
+		}
+	}
+
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
@@ -746,7 +765,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	opts := BucketOptions{
+	opts := MakeBucketOptions{
 		Location:    location,
 		LockEnabled: objectLockEnabled,
 		ForceCreate: forceCreate,
@@ -765,7 +784,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+						Force:      false,
+						NoRecreate: true,
+						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+					})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -1165,7 +1188,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1228,7 +1251,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
@@ -1305,7 +1328,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
+		Force:      forceDelete,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+	}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1472,7 +1498,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle-handlers.go

@@ -62,7 +62,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -163,7 +163,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle.go

@@ -560,7 +560,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 	}
 	// Check if bucket exists.
 	if !r.OutputLocation.IsEmpty() {
-		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
+		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil {
 			return err
 		}
 		if r.OutputLocation.S3.Prefix == "" {

cmd/bucket-metadata.go

@@ -105,10 +105,8 @@ type BucketMetadata struct {
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
 func newBucketMetadata(name string) BucketMetadata {
-	now := UTCNow()
 	return BucketMetadata{
-		Name:    name,
-		Created: now,
+		Name: name,
 		notificationConfig: &event.Config{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
@@ -121,6 +119,17 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 
+// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
+// creation time of bucket on this cluster in all other cases.
+func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
+	if b.Created.IsZero() {
+		b.Created = UTCNow()
+	}
+	if !createdAt.IsZero() {
+		b.Created = createdAt.UTC()
+	}
+}
+
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
@@ -160,17 +169,19 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
-
+	if err == nil {
+		b.defaultTimestamps()
+	}
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
 	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
 		return b, err
 	}
 
 	// migrate unencrypted remote targets
-	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
+	if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
 		return b, err
 	}
-	b.defaultTimestamps()
+
 	return b, nil
 }
 
@@ -345,6 +356,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.BucketTargetsConfigJSON = configData
 		}
 	}
+	b.defaultTimestamps()
 
 	if err := b.Save(ctx, objectAPI); err != nil {
 		return err

cmd/bucket-notification-handlers.go

@@ -60,7 +60,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -132,7 +132,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-policy-handlers.go

@@ -61,7 +61,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -191,7 +191,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers_test.go

@@ -128,7 +128,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 			defer wg.Done()
 			// Sync start.
 			<-start
-			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					t.Logf("unexpected error: %T: %v", err, err)
 					return
@@ -163,7 +163,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T,
 ) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 

cmd/bucket-replication-handlers.go

@@ -55,7 +55,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -118,7 +118,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -157,7 +157,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -198,7 +198,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -267,7 +267,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -356,7 +356,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-replication.go

@@ -1492,7 +1492,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), string(replicationSubsystem))
 	}
 }
 
@@ -1518,7 +1518,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
 	}
 }
 
@@ -1555,7 +1555,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
 	}
 }
 
@@ -1805,6 +1805,7 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 	objInfo.VersionPurgeStatusInternal = ""
 	objInfo.ReplicationStatus = ""
 	objInfo.VersionPurgeStatus = ""
+	delete(objInfo.UserDefined, xhttp.AmzBucketReplicationStatus)
 	resyncdsc := mustReplicate(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(objInfo, replication.ExistingObjectReplicationType, ObjectOptions{}))
 	dsc = &resyncdsc
 	return c.resync(oi, dsc, tgtStatuses)
@@ -1991,7 +1992,7 @@ func (p *ReplicationPool) updateResyncStatus(ctx context.Context, objectAPI Obje
 				if updt {
 					brs.LastUpdate = now
 					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to disk for %s - %w", bucket, err))
+						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to drive for %s - %w", bucket, err))
 						continue
 					}
 				}

cmd/bucket-versioning-handler.go

@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/common-main.go

@@ -264,7 +264,7 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
-	api := operations.NewConsoleAPI(swaggerSpec, buildOpenIDConsoleConfig())
+	api := operations.NewConsoleAPI(swaggerSpec)
 
 	if !serverDebugLog {
 		// Disable console logging if server debug log is not enabled
@@ -275,6 +275,12 @@ func initConsoleServer() (*restapi.Server, error) {
 		api.Logger = noLog
 	}
 
+	// Pass in console application config. This needs to happen before the
+	// ConfigureAPI() call.
+	restapi.GlobalMinIOConfig = restapi.MinIOConfig{
+		OpenIDProviders: buildOpenIDConsoleConfig(),
+	}
+
 	server := restapi.NewServer(api)
 	// register all APIs
 	server.ConfigureAPI()
@@ -502,8 +508,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 			Skip: true,
 		}, nil
 	}
-	const envSeparator = "="
-	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
 	if len(envTokens) != 2 {
 		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
 	}
@@ -529,7 +534,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 // the environment values from a file, in the form "key, value".
 // in a structured form.
 func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
-	f, err := os.Open(envConfigFile)
+	f, err := Open(envConfigFile)
 	if err != nil {
 		return nil, err
 	}
@@ -824,43 +829,56 @@ func handleKMSConfig() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		// Manually load the certificate and private key into memory.
-		// We need to check whether the private key is encrypted, and
-		// if so, decrypt it using the user-provided password.
-		certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
-		}
-		keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
-		}
-		privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
-		if len(rest) != 0 {
-			logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
-		}
-		if x509.IsEncryptedPEMBlock(privateKeyPEM) {
-			keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
-			if err != nil {
-				logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
-			}
-			keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
-		}
-		certificate, err := tls.X509KeyPair(certBytes, keyBytes)
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
-		}
 		rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
 		if err != nil {
 			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
+		loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
+			// Manually load the certificate and private key into memory.
+			// We need to check whether the private key is encrypted, and
+			// if so, decrypt it using the user-provided password.
+			certBytes, err := os.ReadFile(certFile)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+			}
+			keyBytes, err := os.ReadFile(keyFile)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+			}
+			privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+			if len(rest) != 0 {
+				return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
+			}
+			if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+				keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+				}
+				keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+			}
+			certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+			}
+			return certificate, nil
+		}
+
+		reloadCertEvents := make(chan tls.Certificate, 1)
+		certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
+		if err != nil {
+			logger.Fatal(err, "Failed to load KES client certificate")
+		}
+		certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
+		certificate.Notify(reloadCertEvents)
+
 		defaultKeyID := env.Get(config.EnvKESKeyName, "")
 		KMS, err := kms.NewWithConfig(kms.Config{
-			Endpoints:    endpoints,
-			DefaultKeyID: defaultKeyID,
-			Certificate:  certificate,
-			RootCAs:      rootCAs,
+			Endpoints:        endpoints,
+			DefaultKeyID:     defaultKeyID,
+			Certificate:      certificate,
+			ReloadCertEvents: reloadCertEvents,
+			RootCAs:          rootCAs,
 		})
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
@@ -910,7 +928,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	// Therefore, we read all filenames in the cert directory and check
 	// for each directory whether it contains a public.crt and private.key.
 	// If so, we try to add it to certificate manager.
-	root, err := os.Open(globalCertsDir.Get())
+	root, err := Open(globalCertsDir.Get())
 	if err != nil {
 		return nil, nil, false, err
 	}
@@ -929,7 +947,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 			continue
 		}
 		if file.Mode()&os.ModeSymlink == os.ModeSymlink {
-			file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
+			file, err = Stat(filepath.Join(root.Name(), file.Name()))
 			if err != nil {
 				// not accessible ignore
 				continue

cmd/config-migrate_test.go

@@ -37,11 +37,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 	globalConfigDir = &ConfigDir{path: rootPath}
 
 	globalObjLayerMutex.Lock()
@@ -74,13 +70,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 // Test if all migrate code returns nil when config file does not
 // exist
 func TestServerConfigMigrateInexistentConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	globalConfigDir = &ConfigDir{path: rootPath}
+	globalConfigDir = &ConfigDir{path: t.TempDir()}
 
 	if err := migrateV2ToV3(); err != nil {
 		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -164,11 +154,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 
 // Test if a config migration from v2 to v33 is successfully done
 func TestServerConfigMigrateV2toV33(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 
@@ -234,11 +220,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateFaultyConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
@@ -331,35 +313,31 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	for i := 3; i <= 17; i++ {
 		// Create a corrupted config file
-		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+		if err := ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
 			0o644); err != nil {
 			t.Fatal("Unexpected error: ", err)
 		}
 
 		// Test different migrate versions and be sure they are returning an error
-		if err = migrateConfig(); err == nil {
+		if err := migrateConfig(); err == nil {
 			t.Fatal("migrateConfig() should fail with a corrupted json")
 		}
 	}
 
 	// Create a corrupted config file for version '2'.
-	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test different migrate versions and be sure they are returning an error
-	if err = migrateConfig(); err == nil {
+	if err := migrateConfig(); err == nil {
 		t.Fatal("migrateConfig() should fail with a corrupted json")
 	}
 }

cmd/data-update-tracker.go

@@ -227,7 +227,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 	for _, drive := range drives {
 
 		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-		f, err := os.Open(cacheFormatPath)
+		f, err := OpenFile(cacheFormatPath, readMode, 0o666)
 		if err != nil {
 			if osIsNotExist(err) {
 				continue

cmd/data-update-tracker_test.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path"
@@ -100,15 +99,11 @@ func TestDataUpdateTracker(t *testing.T) {
 
 	dut.Current.bf = dut.newBloomFilter()
 
-	tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
+	tmpDir := t.TempDir()
+	err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	dut.start(ctx, tmpDir)

cmd/data-usage_test.go

@@ -35,12 +35,8 @@ type usageTestFile struct {
 }
 
 func TestDataUsageUpdate(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	const bucket = "bucket"
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
@@ -251,12 +247,8 @@ func TestDataUsageUpdate(t *testing.T) {
 }
 
 func TestDataUsageUpdatePrefix(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	scannerSleeper.Update(0, 0)
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "bucket/rootfile", size: 10000},
 		{name: "bucket/rootfile2", size: 10000},
@@ -537,12 +529,8 @@ func generateUsageTestFiles(t *testing.T, base, bucket string, nFolders, nFiles,
 }
 
 func TestDataUsageCacheSerialize(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	const bucket = "abucket"
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},

cmd/disk-cache-backend.go

@@ -624,7 +624,7 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
 		return err
 	}
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|os.O_CREATE|writeMode, 0o666)
 	if err != nil {
 		return err
 	}
@@ -687,7 +687,7 @@ func (c *diskCache) updateMetadata(ctx context.Context, bucket, object, etag str
 	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
 		return err
 	}
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|writeMode, 0o666)
 	if err != nil {
 		return err
 	}
@@ -1275,12 +1275,12 @@ func (c *diskCache) NewMultipartUpload(ctx context.Context, bucket, object, uID
 
 	cachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(cachePath, uploadID)
-	if err := os.MkdirAll(uploadIDDir, 0o777); err != nil {
+	if err := mkdirAll(uploadIDDir, 0o777); err != nil {
 		return uploadID, err
 	}
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
 
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|os.O_CREATE|writeMode, 0o666)
 	if err != nil {
 		return uploadID, err
 	}
@@ -1386,7 +1386,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload
 	defer uploadLock.Unlock(ulkctx.Cancel)
 
 	metaPath := pathJoin(uploadDir, cacheMetaJSONFile)
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|writeMode, 0o666)
 	if err != nil {
 		return err
 	}
@@ -1465,7 +1465,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI
 func (c *diskCache) uploadIDExists(bucket, object, uploadID string) (err error) {
 	mpartCachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(mpartCachePath, uploadID)
-	if _, err := os.Stat(uploadIDDir); err != nil {
+	if _, err := Stat(uploadIDDir); err != nil {
 		return err
 	}
 	return nil
@@ -1564,7 +1564,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	uploadMeta.Hits++
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
 
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|os.O_CREATE|writeMode, 0o666)
 	if err != nil {
 		return oi, err
 	}
@@ -1634,7 +1634,7 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir), func(shaDir string, typ os.FileMode) error {
 				return readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 					uploadIDPath := pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir, uploadIDDir)
-					fi, err := os.Stat(uploadIDPath)
+					fi, err := Stat(uploadIDPath)
 					if err != nil {
 						return nil
 					}
@@ -1649,8 +1649,8 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheWritebackDir), func(shaDir string, typ os.FileMode) error {
 				wbdir := pathJoin(c.dir, minioMetaBucket, cacheWritebackDir, shaDir)
 				cachedir := pathJoin(c.dir, shaDir)
-				if _, err := os.Stat(cachedir); os.IsNotExist(err) {
-					fi, err := os.Stat(wbdir)
+				if _, err := Stat(cachedir); os.IsNotExist(err) {
+					fi, err := Stat(wbdir)
 					if err != nil {
 						return nil
 					}

cmd/disk-cache.go

@@ -607,12 +607,12 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 				warningMsg = fmt.Sprintf("Invalid cache dir %s err : %s", dir, err.Error())
 			}
 			if rootDsk {
-				warningMsg = fmt.Sprintf("cache dir cannot be part of root disk: %s", dir)
+				warningMsg = fmt.Sprintf("cache dir cannot be part of root drive: %s", dir)
 			}
 		}
 
 		if err := checkAtimeSupport(dir); err != nil {
-			return nil, false, fmt.Errorf("Atime support required for disk caching, atime check failed with %w", err)
+			return nil, false, fmt.Errorf("Atime support required for drive caching, atime check failed with %w", err)
 		}
 
 		cache, err := newDiskCache(ctx, dir, config)
@@ -622,7 +622,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		caches = append(caches, cache)
 	}
 	if warningMsg != "" {
-		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root disk for disk caching is deprecated: %s", warningMsg)))
+		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root drive for drive caching is deprecated: %s", warningMsg)))
 	}
 	return caches, migrating, nil
 }

cmd/dummy-handlers.go

@@ -52,7 +52,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -84,7 +84,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -117,7 +117,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -151,7 +151,7 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -189,7 +189,7 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/endpoint-ellipses.go

@@ -156,7 +156,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	setCounts := possibleSetCounts(commonSize)
 	if len(setCounts) == 0 {
-		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of drives %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
@@ -183,7 +183,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns)
 
 		if len(setCounts) == 0 {
-			msg := fmt.Sprintf("No symmetric distribution detected with input endpoints provided %s, disks %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes)
+			msg := fmt.Sprintf("No symmetric distribution detected with input endpoints provided %s, drives %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes)
 			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 		}
 
@@ -193,7 +193,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of drives %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 

cmd/endpoint.go

@@ -292,6 +292,19 @@ func (l EndpointServerPools) LocalDisksPaths() []string {
 	return disks
 }
 
+// NLocalDisksPathsPerPool returns the disk paths of the local disks per pool
+func (l EndpointServerPools) NLocalDisksPathsPerPool() []int {
+	localDisksCount := make([]int, len(l))
+	for i, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				localDisksCount[i]++
+			}
+		}
+	}
+	return localDisksCount
+}
+
 // FirstLocal returns true if the first endpoint is local.
 func (l EndpointServerPools) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
@@ -302,7 +315,7 @@ func (l EndpointServerPools) HTTPS() bool {
 	return l[0].Endpoints.HTTPS()
 }
 
-// NEndpoints - returns all nodes count
+// NEndpoints - returns number of endpoints
 func (l EndpointServerPools) NEndpoints() (count int) {
 	for _, ep := range l {
 		count += len(ep.Endpoints)

cmd/erasure-bucket.go

@@ -35,7 +35,7 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 // Bucket operations
 
 // MakeBucket - make a bucket.
-func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Verify if bucket is valid.
@@ -94,7 +94,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
 }
 
 // getBucketInfo - returns the BucketInfo from one of the load balanced disks.
-func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) {
+func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -108,9 +108,17 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 			}
 			volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
 			if err != nil {
+				if opts.Deleted {
+					dvi, derr := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucketName))
+					if derr != nil {
+						return err
+					}
+					bucketsInfo[index] = BucketInfo{Name: bucketName, Deleted: dvi.Created}
+					return nil
+				}
 				return err
 			}
-			bucketsInfo[index] = BucketInfo(volInfo)
+			bucketsInfo[index] = BucketInfo{Name: volInfo.Name, Created: volInfo.Created}
 			return nil
 		}, index)
 	}
@@ -131,8 +139,8 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
-func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketInfo, err := er.getBucketInfo(ctx, bucket)
+func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
+	bucketInfo, err := er.getBucketInfo(ctx, bucket, opts)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
 	}
@@ -191,12 +199,58 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts D
 		if purgedDangling {
 			err = nil
 		}
-
+		if opts.SRDeleteOp == MarkDelete {
+			er.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
 	}
 
 	return toObjectErr(err, bucket)
 }
 
+// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
+// syncs the delete to peers
+func (er erasureObjects) markDelete(ctx context.Context, bucket, prefix string) error {
+	storageDisks := er.getDisks()
+	g := errgroup.WithNErrs(len(storageDisks))
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		if storageDisks[index] == nil {
+			continue
+		}
+		g.Go(func() error {
+			if err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, prefix)); err != nil {
+				if errors.Is(err, errVolumeExists) {
+					return nil
+				}
+				return err
+			}
+			return nil
+		}, index)
+	}
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
+	return toObjectErr(err, bucket)
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers OR on a new MakeBucket call.
+func (er erasureObjects) purgeDelete(ctx context.Context, bucket, prefix string) error {
+	storageDisks := er.getDisks()
+	g := errgroup.WithNErrs(len(storageDisks))
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] != nil {
+				return storageDisks[index].DeleteVol(ctx, pathJoin(bucket, prefix), true)
+			}
+			return errDiskNotFound
+		}, index)
+	}
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
+	return toObjectErr(err, bucket)
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (er erasureObjects) IsNotificationSupported() bool {
 	return true

cmd/erasure-decode_test.go

@@ -85,19 +85,17 @@ var erasureDecodeTests = []struct {
 
 func TestErasureDecode(t *testing.T) {
 	for i, test := range erasureDecodeTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		disks := setup.disks
 		data := make([]byte, test.data)
 		if _, err = io.ReadFull(crand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
 
@@ -113,11 +111,9 @@ func TestErasureDecode(t *testing.T) {
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
 		}
 		if n != test.data {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file", i)
 		}
 		for i, w := range writers {
@@ -195,7 +191,6 @@ func TestErasureDecode(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }
 
@@ -210,12 +205,11 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 	dataBlocks := 7
 	parityBlocks := 7
 	blockSize := int64(1 * humanize.MiByte)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	setup, err := newErasureTestSetup(t, dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-	defer setup.Remove()
 	disks := setup.disks
 	erasure, err := NewErasure(context.Background(), dataBlocks, parityBlocks, blockSize)
 	if err != nil {
@@ -288,11 +282,10 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 // Benchmarks
 
 func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV2)
+	setup, err := newErasureTestSetup(b, data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
-	defer setup.Remove()
 	disks := setup.disks
 	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2)
 	if err != nil {

cmd/erasure-encode_test.go

@@ -87,21 +87,19 @@ var erasureEncodeTests = []struct {
 
 func TestErasureEncode(t *testing.T) {
 	for i, test := range erasureEncodeTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 
 		data := make([]byte, test.data)
 		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
 		writers := make([]io.Writer, len(disks))
@@ -160,18 +158,16 @@ func TestErasureEncode(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }
 
 // Benchmarks
 
 func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV2)
+	setup, err := newErasureTestSetup(b, data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
-	defer setup.Remove()
 	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)

cmd/erasure-errors.go

@@ -20,10 +20,10 @@ package cmd
 import "errors"
 
 // errErasureReadQuorum - did not meet read quorum.
-var errErasureReadQuorum = errors.New("Read failed. Insufficient number of disks online")
+var errErasureReadQuorum = errors.New("Read failed. Insufficient number of drives online")
 
 // errErasureWriteQuorum - did not meet write quorum.
-var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of disks online")
+var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of drives online")
 
 // errNoHealRequired - returned when healing is attempted on a previously healed disks.
 var errNoHealRequired = errors.New("No healing is required")

cmd/erasure-heal_test.go

@@ -66,23 +66,21 @@ func TestErasureHeal(t *testing.T) {
 	for i, test := range erasureHealTests {
 		if test.offDisks < test.badStaleDisks {
 			// test case sanity check
-			t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
+			t.Fatalf("Test %d: Bad test case - number of stale drives cannot be less than number of badstale drives", i)
 		}
 
 		// create some test data
-		setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to setup Erasure environment: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		data := make([]byte, test.size)
 		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
@@ -93,7 +91,6 @@ func TestErasureHeal(t *testing.T) {
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 
@@ -156,6 +153,5 @@ func TestErasureHeal(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }

cmd/erasure-healing-common_test.go

@@ -176,7 +176,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -260,7 +260,7 @@ func TestListOnlineDisks(t *testing.T) {
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
-					t.Fatalf("disk (%v) with part.1 missing is not a disk with available data",
+					t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
 						erasureDisks[tamperedIndex])
 				}
 			}
@@ -351,7 +351,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -446,7 +446,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
-					t.Fatalf("disk (%v) with part.1 missing is not a disk with available data",
+					t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
 						erasureDisks[tamperedIndex])
 				}
 			}
@@ -472,7 +472,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	z := obj.(*erasureServerPools)
 	s := z.serverPools[0].sets[0]
 	erasureDisks := s.getDisks()
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -506,7 +506,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 
 	for diskIndex, disk := range filteredDisks {
@@ -515,7 +515,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		}
 
 		if disk == nil {
-			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+			t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 		}
 	}
 
@@ -528,14 +528,14 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 	for diskIndex, disk := range filteredDisks {
 		if diskIndex == 0 && disk != nil {
-			t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+			t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 		}
 		if diskIndex != 0 && disk == nil {
-			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+			t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 		}
 	}
 	partsMetadata[0] = partsMetadataBackup // Revert before going to the next test
@@ -549,14 +549,14 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 	for diskIndex, disk := range filteredDisks {
 		if diskIndex == 1 && disk != nil {
-			t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+			t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 		}
 		if diskIndex != 1 && disk == nil {
-			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+			t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 		}
 	}
 	partsMetadata[1] = partsMetadataBackup // Revert before going to the next test
@@ -586,23 +586,23 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 
 	for diskIndex, disk := range filteredDisks {
 		if _, ok := diskFailures[diskIndex]; ok {
 			if disk != nil {
-				t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+				t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 			}
 			if errs[diskIndex] == nil {
-				t.Errorf("Expected error not received, diskIndex: %d", diskIndex)
+				t.Errorf("Expected error not received, driveIndex: %d", diskIndex)
 			}
 		} else {
 			if disk == nil {
-				t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+				t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 			}
 			if errs[diskIndex] != nil {
-				t.Errorf("Unexpected error, %s, diskIndex: %d", errs[diskIndex], diskIndex)
+				t.Errorf("Unexpected error, %s, driveIndex: %d", errs[diskIndex], diskIndex)
 			}
 
 		}

cmd/erasure-healing.go

@@ -243,7 +243,7 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
+func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo, doinline bool) bool {
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
@@ -256,6 +256,10 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 			// always check first.
 			return true
 		}
+		if doinline {
+			// convert small files to 'inline'
+			return true
+		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -275,10 +279,6 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
-	if !opts.DryRun {
-		defer NSUpdated(bucket, object)
-	}
-
 	dryRun := opts.DryRun
 	scanMode := opts.ScanMode
 
@@ -346,6 +346,26 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
+	var erasure Erasure
+	var recreate bool
+	if !latestMeta.Deleted && !latestMeta.IsRemote() {
+		// Initialize erasure coding
+		erasure, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks,
+			latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
+		if err != nil {
+			return result, err
+		}
+
+		// Is only 'true' if the opts.Recreate is true and
+		// the object shardSize < smallFileThreshold do not
+		// set this to 'true' arbitrarily and must be only
+		// 'true' with caller ask.
+		recreate = (opts.Recreate &&
+			!latestMeta.InlineData() &&
+			len(latestMeta.Parts) == 1 &&
+			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
+	}
+
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
@@ -372,7 +392,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta, recreate) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
@@ -426,9 +446,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// When disk to heal count is greater than parity blocks we should simply error out.
-		err := fmt.Errorf("more disks are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
+		err := fmt.Errorf("more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
 		logger.LogIf(ctx, err)
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
@@ -482,18 +502,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	}
 
 	var inlineBuffers []*bytes.Buffer
-	if latestMeta.InlineData() {
-		inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
-	}
-
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		// Heal each part. erasureHealFile() will write the healed
-		// part to .minio/tmp/uuid/ which needs to be renamed later to
-		// the final location.
-		erasure, err := NewErasure(ctx, latestMeta.Erasure.DataBlocks,
-			latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
-		if err != nil {
-			return result, err
+		if latestMeta.InlineData() || recreate {
+			inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 		}
 
 		erasureInfo := latestMeta.Erasure
@@ -529,6 +540,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
+
+			// Heal each part. erasure.Heal() will write the healed
+			// part to .minio/tmp/uuid/ which needs to be renamed
+			// later to the final location.
 			err = erasure.Heal(ctx, writers, readers, partSize)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
@@ -560,6 +575,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				})
 				if len(inlineBuffers) > 0 && inlineBuffers[i] != nil {
 					partsMetadata[i].Data = inlineBuffers[i].Bytes()
+					partsMetadata[i].SetInlineData()
 				} else {
 					partsMetadata[i].Data = nil
 				}
@@ -567,7 +583,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 			// If all disks are having errors, we give up.
 			if disksToHealCount == 0 {
-				return result, fmt.Errorf("all disks had write errors, unable to heal %s/%s", bucket, object)
+				return result, fmt.Errorf("all drives had write errors, unable to heal %s/%s", bucket, object)
 			}
 
 		}
@@ -591,8 +607,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			return result, err
 		}
 
-		// Remove any remaining parts from outdated disks from before transition.
-		if partsMetadata[i].IsRemote() {
+		// - Remove any parts from healed disks after its been inlined.
+		// - Remove any remaining parts from outdated disks from before transition.
+		if recreate || partsMetadata[i].IsRemote() {
 			rmDataDir := partsMetadata[i].DataDir
 			disk.DeleteVol(ctx, pathJoin(bucket, encodeDirObject(object), rmDataDir), true)
 		}

cmd/erasure-healing_test.go

@@ -236,7 +236,7 @@ func TestHealing(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -359,7 +359,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	object := getRandomObjectName()
 	data := bytes.Repeat([]byte("a"), 128*1024)
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -554,7 +554,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -685,7 +685,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -861,7 +861,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1004,7 +1004,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1161,7 +1161,7 @@ func TestHealObjectErasure(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1258,7 +1258,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	object := "empty-dir/"
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1359,7 +1359,7 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 			var opts ObjectOptions
 
-			err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				t.Fatalf("Failed to make a bucket - %v", err)
 			}

cmd/erasure-metadata-utils.go

@@ -275,7 +275,7 @@ func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []Stora
 // the corresponding error in errs slice is not nil
 func evalDisks(disks []StorageAPI, errs []error) []StorageAPI {
 	if len(errs) != len(disks) {
-		logger.LogIf(GlobalContext, errors.New("unexpected disks/errors slice length"))
+		logger.LogIf(GlobalContext, errors.New("unexpected drives/errors slice length"))
 		return nil
 	}
 	newDisks := make([]StorageAPI, len(disks))

cmd/erasure-multipart.go

@@ -1159,6 +1159,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, toObjectErr(err, bucket, object)
 	}
 
+	defer NSUpdated(bucket, object)
+
 	// Check if there is any offline disk and add it to the MRF list
 	for _, disk := range onlineDisks {
 		if disk != nil && disk.IsOnline() {

cmd/erasure-object.go

@@ -534,6 +534,11 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		return metaFileInfos, errs
 	}
 
+	versionID := lfi.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+
 	for index := range metadataArray {
 		if metadataArray[index] == nil {
 			continue
@@ -541,21 +546,15 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 		// make sure to preserve this for diskmtime based healing bugfix.
 		diskMTime := metaFileInfos[index].DiskMTime
-		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, "")
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID)
 		if errs[index] != nil {
 			continue
 		}
 
-		if metaFileInfos[index].IsValid() && metaFileInfos[index].ModTime.Equal(lfi.ModTime) {
-			versionID := metaFileInfos[index].VersionID
-			if versionID == "" {
-				versionID = nullVersionID
-			}
-			if readData {
-				metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
-			}
-			metaFileInfos[index].DiskMTime = diskMTime
+		if readData {
+			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
 		}
+		metaFileInfos[index].DiskMTime = diskMTime
 	}
 
 	// Return all the metadata.
@@ -687,8 +686,6 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
 func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
-	defer NSUpdated(dstBucket, dstEntry)
-
 	g := errgroup.WithNErrs(len(disks))
 
 	fvID := mustGetUUID()
@@ -1141,6 +1138,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
+	defer NSUpdated(bucket, object)
+
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick

cmd/erasure-object_test.go

@@ -53,7 +53,7 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	defer objLayer.Shutdown(context.Background())
 	defer removeRoots(disks)
 
-	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +101,7 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	}
 	defer xl.Shutdown(context.Background())
 
-	err = xl.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = xl.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +155,7 @@ func TestDeleteObjectsVersioned(t *testing.T) {
 		{bucketName, "dir/obj1"},
 	}
 
-	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+	err = obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{
 		VersioningEnabled: true,
 	})
 	if err != nil {
@@ -244,7 +244,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		{bucketName, "obj_4"},
 	}
 
-	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, BucketOptions{})
+	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -305,7 +305,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -374,7 +374,7 @@ func TestErasureDeleteObjectDiskNotFoundErasure4(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -434,7 +434,7 @@ func TestErasureDeleteObjectDiskNotFoundErr(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +505,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -614,7 +614,7 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +692,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -755,7 +755,7 @@ func TestPutObjectNoQuorumSmall(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -824,7 +824,7 @@ func TestPutObjectSmallInlineData(t *testing.T) {
 	object := "object"
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -892,7 +892,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	ctx, cancel := context.WithCancel(GlobalContext)
 	defer cancel()
 
-	err := obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -1073,10 +1073,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	// Create a backend with 4 disks named disk{1...4}, this name convention
 	// because we will unzip some object data from a sample archive.
 	const numDisks = 4
-	path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
+	path := t.TempDir()
 
 	var fsDirs []string
 	for i := 1; i <= numDisks; i++ {
@@ -1094,7 +1091,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Create a testbucket
-	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1160,7 +1157,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 
 	for i, testCase := range testCases {
 		// Step 1: create a bucket
-		err = z.MakeBucketWithLocation(ctx, testCase.bucket, BucketOptions{VersioningEnabled: testCase.versioned})
+		err = z.MakeBucketWithLocation(ctx, testCase.bucket, MakeBucketOptions{VersioningEnabled: testCase.versioned})
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create a bucket: %v", i+1, err)
 		}

cmd/erasure-server-pool-decom.go

@@ -457,7 +457,7 @@ func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSet
 	now := UTCNow()
 	if now.Sub(p.Pools[idx].LastUpdate) >= duration {
 		if serverDebugLog {
-			console.Debugf("decommission: persisting poolMeta on disk: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx])
+			console.Debugf("decommission: persisting poolMeta on drive: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx])
 		}
 		p.Pools[idx].LastUpdate = now
 		if err := p.save(ctx, pools); err != nil {
@@ -677,7 +677,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		set := set
 		disks := set.getOnlineDisks()
 		if len(disks) == 0 {
-			logger.LogIf(GlobalContext, fmt.Errorf("no online disks found for set with endpoints %s",
+			logger.LogIf(GlobalContext, fmt.Errorf("no online drives found for set with endpoints %s",
 				set.getEndpoints()))
 			continue
 		}
@@ -1119,7 +1119,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return errInvalidArgument
 	}
 
-	buckets, err := z.ListBuckets(ctx)
+	buckets, err := z.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -1157,7 +1157,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		pathJoin(minioMetaBucket, bucketMetaPrefix),
 	} {
 		var bucketExists BucketExists
-		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, MakeBucketOptions{}); err != nil {
 			if !errors.As(err, &bucketExists) {
 				return err
 			}

cmd/erasure-server-pool.go

@@ -533,9 +533,11 @@ func (z *erasureServerPools) BackendInfo() (b madmin.BackendInfo) {
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
 
 	// Data blocks can vary per pool, but parity is same.
-	for _, setDriveCount := range z.SetDriveCounts() {
+	for i, setDriveCount := range z.SetDriveCounts() {
 		b.StandardSCData = append(b.StandardSCData, setDriveCount-scParity)
 		b.RRSCData = append(b.RRSCData, setDriveCount-rrSCParity)
+		b.DrivesPerSet = append(b.DrivesPerSet, setDriveCount)
+		b.TotalSets = append(b.TotalSets, z.serverPools[i].setCount)
 	}
 
 	b.StandardSCParity = scParity
@@ -613,7 +615,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 	var results []dataUsageCache
 	var firstErr error
 
-	allBuckets, err := z.ListBuckets(ctx)
+	allBuckets, err := z.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -718,7 +720,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 // MakeBucketWithLocation - creates a new bucket across all serverPools simultaneously
 // even if one of the sets fail to create buckets, we proceed all the successful
 // operations.
-func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	// Lock the bucket name before creating.
@@ -758,6 +760,7 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 
 	// If it doesn't exist we get a new, so ignore errors
 	meta := newBucketMetadata(bucket)
+	meta.SetCreatedAt(opts.CreatedAt)
 	if opts.LockEnabled {
 		meta.VersioningConfigXML = enabledBucketVersioningConfig
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
@@ -1539,9 +1542,9 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 }
 
 // GetBucketInfo - returns bucket info from one of the erasure coded serverPools.
-func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
+func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
 	if z.SinglePool() {
-		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket)
+		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket, opts)
 		if err != nil {
 			return bucketInfo, err
 		}
@@ -1552,7 +1555,7 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (
 		return bucketInfo, nil
 	}
 	for _, pool := range z.serverPools {
-		bucketInfo, err = pool.GetBucketInfo(ctx, bucket)
+		bucketInfo, err = pool.GetBucketInfo(ctx, bucket, opts)
 		if err != nil {
 			if isErrBucketNotFound(err) {
 				continue
@@ -1626,7 +1629,11 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 
 	// Purge the entire bucket metadata entirely.
 	z.renameAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
-
+	// If site replication is configured, hold on to deleted bucket state until sites sync
+	switch opts.SRDeleteOp {
+	case MarkDelete:
+		z.markDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+	}
 	// Success.
 	return nil
 }
@@ -1644,6 +1651,27 @@ func (z *erasureServerPools) renameAll(ctx context.Context, bucket, prefix strin
 	}
 }
 
+// markDelete will create a directory of deleted bucket in .minio.sys/buckets/.deleted across all disks
+// in situations where the deleted bucket needs to be held on to until all sites are in sync for
+// site replication
+func (z *erasureServerPools) markDelete(ctx context.Context, bucket, prefix string) {
+	for _, servers := range z.serverPools {
+		for _, set := range servers.sets {
+			set.markDelete(ctx, bucket, prefix)
+		}
+	}
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers.
+func (z *erasureServerPools) purgeDelete(ctx context.Context, bucket, prefix string) {
+	for _, servers := range z.serverPools {
+		for _, set := range servers.sets {
+			set.purgeDelete(ctx, bucket, prefix)
+		}
+	}
+}
+
 // This function is used to undo a successful DeleteBucket operation.
 func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools []*erasureSets, errs []error) {
 	g := errgroup.WithNErrs(len(serverPools))
@@ -1653,7 +1681,7 @@ func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools
 		index := index
 		g.Go(func() error {
 			if errs[index] == nil {
-				return serverPools[index].MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+				return serverPools[index].MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			}
 			return nil
 		}, index)
@@ -1665,15 +1693,15 @@ func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools
 // List all buckets from one of the serverPools, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
-func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	if z.SinglePool() {
-		buckets, err = z.serverPools[0].ListBuckets(ctx)
+		buckets, err = z.serverPools[0].ListBuckets(ctx, opts)
 	} else {
 		for idx, pool := range z.serverPools {
 			if z.IsSuspended(idx) {
 				continue
 			}
-			buckets, err = pool.ListBuckets(ctx)
+			buckets, err = pool.ListBuckets(ctx, opts)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
@@ -1685,9 +1713,9 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
 		return nil, err
 	}
 	for i := range buckets {
-		meta, err := globalBucketMetadataSys.Get(buckets[i].Name)
+		createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
 		if err == nil {
-			buckets[i].Created = meta.Created
+			buckets[i].Created = createdAt
 		}
 	}
 	return buckets, nil
@@ -1741,7 +1769,9 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 	}
 
 	// Attempt heal on the bucket metadata, ignore any failures
-	defer z.HealObject(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), "", opts)
+	hopts := opts
+	hopts.Recreate = false
+	defer z.HealObject(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), "", hopts)
 
 	for _, pool := range z.serverPools {
 		result, err := pool.HealBucket(ctx, bucket, opts)
@@ -1877,7 +1907,7 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 
 	disks, _ := set.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		return errors.New("listAndHeal: No non-healing disks found")
+		return errors.New("listAndHeal: No non-healing drives found")
 	}
 
 	// How to resolve partial results.
@@ -2068,7 +2098,7 @@ func (z *erasureServerPools) getPoolAndSet(id string) (poolIdx, setIdx, diskIdx
 			}
 		}
 	}
-	return -1, -1, -1, fmt.Errorf("DiskID(%s) %w", id, errDiskNotFound)
+	return -1, -1, -1, fmt.Errorf("DriveID(%s) %w", id, errDiskNotFound)
 }
 
 // HealthOptions takes input options to return sepcific information

cmd/erasure-sets.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"reflect"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -128,10 +129,10 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		if errors.Is(err, errUnformattedDisk) {
 			info, derr := disk.DiskInfo(context.TODO())
 			if derr != nil && info.RootDisk {
-				return nil, nil, fmt.Errorf("Disk: %s is a root disk", disk)
+				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
 		}
-		return nil, nil, fmt.Errorf("Disk: %s returned %w", disk, err) // make sure to '%w' to wrap the error
+		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
 	return disk, format, nil
@@ -146,7 +147,7 @@ func findDiskIndexByDiskID(refFormat *formatErasureV3, diskID string) (int, int,
 		return -1, -1, errDiskNotFound
 	}
 	if diskID == offlineDiskUUID {
-		return -1, -1, fmt.Errorf("diskID: %s is offline", diskID)
+		return -1, -1, fmt.Errorf("DriveID: %s is offline", diskID)
 	}
 	for i := 0; i < len(refFormat.Erasure.Sets); i++ {
 		for j := 0; j < len(refFormat.Erasure.Sets[0]); j++ {
@@ -156,7 +157,7 @@ func findDiskIndexByDiskID(refFormat *formatErasureV3, diskID string) (int, int,
 		}
 	}
 
-	return -1, -1, fmt.Errorf("diskID: %s not found", diskID)
+	return -1, -1, fmt.Errorf("DriveID: %s not found", diskID)
 }
 
 // findDiskIndex - returns the i,j'th position of the input `format` against the reference
@@ -169,7 +170,7 @@ func findDiskIndex(refFormat, format *formatErasureV3) (int, int, error) {
 	}
 
 	if format.Erasure.This == offlineDiskUUID {
-		return -1, -1, fmt.Errorf("diskID: %s is offline", format.Erasure.This)
+		return -1, -1, fmt.Errorf("DriveID: %s is offline", format.Erasure.This)
 	}
 
 	for i := 0; i < len(refFormat.Erasure.Sets); i++ {
@@ -180,7 +181,7 @@ func findDiskIndex(refFormat, format *formatErasureV3) (int, int, error) {
 		}
 	}
 
-	return -1, -1, fmt.Errorf("diskID: %s not found", format.Erasure.This)
+	return -1, -1, fmt.Errorf("DriveID: %s not found", format.Erasure.This)
 }
 
 // connectDisks - attempt to connect all the endpoints, loads format
@@ -238,7 +239,7 @@ func (s *erasureSets) connectDisks() {
 			s.erasureDisksMu.Lock()
 			if currentDisk := s.erasureDisks[setIndex][diskIndex]; currentDisk != nil {
 				if !reflect.DeepEqual(currentDisk.Endpoint(), disk.Endpoint()) {
-					err = fmt.Errorf("Detected unexpected disk ordering refusing to use the disk: expecting %s, found %s, refusing to use the disk",
+					err = fmt.Errorf("Detected unexpected drive ordering refusing to use the drive: expecting %s, found %s, refusing to use the drive",
 						currentDisk.Endpoint(), disk.Endpoint())
 					printEndpointError(endpoint, err, false)
 					disk.Close()
@@ -299,7 +300,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 			return
 		case <-monitor.C:
 			if serverDebugLog {
-				console.Debugln("running disk monitoring")
+				console.Debugln("running drive monitoring")
 			}
 
 			s.connectDisks()
@@ -445,7 +446,7 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 						return
 					}
 					if m != i || n != j {
-						logger.LogIf(ctx, fmt.Errorf("Detected unexpected disk ordering refusing to use the disk - poolID: %s, found disk mounted at (set=%s, disk=%s) expected mount at (set=%s, disk=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
+						logger.LogIf(ctx, fmt.Errorf("Detected unexpected drive ordering refusing to use the drive - poolID: %s, found drive mounted at (set=%s, drive=%s) expected mount at (set=%s, drive=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
 						s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
 						return
 					}
@@ -689,7 +690,7 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 
 // MakeBucketLocation - creates a new bucket across all sets simultaneously,
 // then return the first encountered error
-func (s *erasureSets) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (s *erasureSets) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	g := errgroup.WithNErrs(len(s.sets))
 
 	// Create buckets in parallel across all sets.
@@ -760,8 +761,8 @@ func (s *erasureSets) getHashedSet(input string) (set *erasureObjects) {
 }
 
 // GetBucketInfo - returns bucket info from one of the erasure coded set.
-func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
-	return s.getHashedSet("").GetBucketInfo(ctx, bucket)
+func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
+	return s.getHashedSet("").GetBucketInfo(ctx, bucket, opts)
 }
 
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
@@ -825,7 +826,7 @@ func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObj
 		index := index
 		g.Go(func() error {
 			if errs[index] == nil {
-				return sets[index].MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+				return sets[index].MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			}
 			return nil
 		}, index)
@@ -837,7 +838,7 @@ func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObj
 // List all buckets from one of the set, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all sets.
-func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+func (s *erasureSets) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	var listBuckets []BucketInfo
 	healBuckets := map[string]VolInfo{}
 	for _, set := range s.sets {
@@ -847,8 +848,34 @@ func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, er
 		}
 	}
 
+	// include deleted buckets in listBuckets output
+	deletedBuckets := map[string]VolInfo{}
+
+	if opts.Deleted {
+		for _, set := range s.sets {
+			// lists all unique buckets across drives.
+			if err := listDeletedBuckets(ctx, set.getDisks(), deletedBuckets, s.defaultParityCount); err != nil {
+				return nil, err
+			}
+		}
+	}
 	for _, v := range healBuckets {
-		listBuckets = append(listBuckets, BucketInfo(v))
+		bi := BucketInfo{
+			Name:    v.Name,
+			Created: v.Created,
+		}
+		if vi, ok := deletedBuckets[v.Name]; ok {
+			bi.Deleted = vi.Created
+		}
+		listBuckets = append(listBuckets, bi)
+	}
+	for _, v := range deletedBuckets {
+		if _, ok := healBuckets[v.Name]; !ok {
+			listBuckets = append(listBuckets, BucketInfo{
+				Name:    v.Name,
+				Deleted: v.Created,
+			})
+		}
 	}
 
 	sort.Slice(listBuckets, func(i, j int) bool {
@@ -858,6 +885,45 @@ func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, er
 	return listBuckets, nil
 }
 
+// listDeletedBuckets lists deleted buckets from all disks.
+func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets map[string]VolInfo, readQuorum int) error {
+	g := errgroup.WithNErrs(len(storageDisks))
+	var mu sync.Mutex
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] == nil {
+				// we ignore disk not found errors
+				return nil
+			}
+			volsInfo, err := storageDisks[index].ListDir(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix), -1)
+			if err != nil {
+				if err == errFileNotFound {
+					return nil
+				}
+				return err
+			}
+			for _, volName := range volsInfo {
+				mu.Lock()
+				if _, ok := delBuckets[volName]; !ok {
+					vi, err := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, volName))
+					if err != nil {
+						return err
+					}
+					bkt := strings.TrimPrefix(vi.Name, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix))
+					bkt = strings.TrimPrefix(bkt, slashSeparator)
+					bkt = strings.TrimSuffix(bkt, slashSeparator)
+					vi.Name = bkt
+					delBuckets[bkt] = vi
+				}
+				mu.Unlock()
+			}
+			return nil
+		}, index)
+	}
+	return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum)
+}
+
 // --- Object Operations ---
 
 // GetObjectNInfo - returns object info and locked object ReadCloser
@@ -1174,7 +1240,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 		if storageDisks[i] != nil && infos[i].RootDisk {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.
-			logger.LogIf(GlobalContext, fmt.Errorf("Disk `%s` is part of root disk, will not be used", storageDisks[i]))
+			logger.LogIf(GlobalContext, fmt.Errorf("Drive `%s` is part of root drive, will not be used", storageDisks[i]))
 			storageDisks[i] = nil
 		}
 	}
@@ -1248,7 +1314,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				continue
 			}
 			if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Disk %s failed to write updated 'format.json': %v", storageDisks[index], err))
+				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}
 		}

cmd/erasure-sets_test.go

@@ -187,7 +187,7 @@ func TestNewErasureSets(t *testing.T) {
 	// Initializes all erasure disks
 	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "", "")
 	if err != nil {
-		t.Fatalf("Unable to format disks for erasure, %s", err)
+		t.Fatalf("Unable to format drives for erasure, %s", err)
 	}
 
 	ep := PoolEndpoints{Endpoints: endpoints}

cmd/erasure-single-drive.go

@@ -108,16 +108,40 @@ func newErasureSingle(ctx context.Context, storageDisk StorageAPI, format *forma
 // List all buckets from one of the set, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all sets.
-func (es *erasureSingle) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+func (es *erasureSingle) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	var listBuckets []BucketInfo
 	healBuckets := map[string]VolInfo{}
 	// lists all unique buckets across drives.
 	if err := listAllBuckets(ctx, []StorageAPI{es.disk}, healBuckets, 0); err != nil {
 		return nil, err
 	}
+	// include deleted buckets in listBuckets output
+	deletedBuckets := map[string]VolInfo{}
 
+	if opts.Deleted {
+		// lists all deleted buckets across drives.
+		if err := listDeletedBuckets(ctx, []StorageAPI{es.disk}, deletedBuckets, 0); err != nil {
+			return nil, err
+		}
+	}
 	for _, v := range healBuckets {
-		listBuckets = append(listBuckets, BucketInfo(v))
+		bi := BucketInfo{
+			Name:    v.Name,
+			Created: v.Created,
+		}
+		if vi, ok := deletedBuckets[v.Name]; ok {
+			bi.Deleted = vi.Created
+		}
+		listBuckets = append(listBuckets, bi)
+	}
+
+	for _, v := range deletedBuckets {
+		if _, ok := healBuckets[v.Name]; !ok {
+			listBuckets = append(listBuckets, BucketInfo{
+				Name:    v.Name,
+				Deleted: v.Created,
+			})
+		}
 	}
 
 	sort.Slice(listBuckets, func(i, j int) bool {
@@ -256,7 +280,7 @@ type renameAllStorager interface {
 
 // Bucket operations
 // MakeBucket - make a bucket.
-func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Lock the bucket name before creating.
@@ -289,6 +313,7 @@ func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket stri
 
 	// If it doesn't exist we get a new, so ignore errors
 	meta := newBucketMetadata(bucket)
+	meta.SetCreatedAt(opts.CreatedAt)
 	if opts.LockEnabled {
 		meta.VersioningConfigXML = enabledBucketVersioningConfig
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
@@ -308,12 +333,17 @@ func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket stri
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
-func (es *erasureSingle) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
+func (es *erasureSingle) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
 	volInfo, err := es.disk.StatVol(ctx, bucket)
 	if err != nil {
+		if opts.Deleted {
+			if dvi, derr := es.disk.StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket)); derr == nil {
+				return BucketInfo{Name: bucket, Deleted: dvi.Created}, nil
+			}
+		}
 		return bi, toObjectErr(err, bucket)
 	}
-	return BucketInfo(volInfo), nil
+	return BucketInfo{Name: volInfo.Name, Created: volInfo.Created}, nil
 }
 
 // DeleteBucket - deletes a bucket.
@@ -326,6 +356,28 @@ func (es *erasureSingle) DeleteBucket(ctx context.Context, bucket string, opts D
 	deleteBucketMetadata(ctx, es, bucket)
 	globalBucketMetadataSys.Remove(bucket)
 
+	if err == nil || errors.Is(err, errVolumeNotFound) {
+		if opts.SRDeleteOp == MarkDelete {
+			es.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
+	}
+	return toObjectErr(err, bucket)
+}
+
+// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
+// syncs the delete to peers
+func (es *erasureSingle) markDelete(ctx context.Context, bucket, prefix string) error {
+	err := es.disk.MakeVol(ctx, pathJoin(bucket, prefix))
+	if err != nil && errors.Is(err, errVolumeExists) {
+		return nil
+	}
+	return toObjectErr(err, bucket)
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers OR on a new MakeBucket call.
+func (es *erasureSingle) purgeDelete(ctx context.Context, bucket, prefix string) error {
+	err := es.disk.DeleteVol(ctx, pathJoin(bucket, prefix), true)
 	return toObjectErr(err, bucket)
 }
 
@@ -1151,6 +1203,8 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
+	defer NSUpdated(bucket, object)
+
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -1256,12 +1310,12 @@ func (es *erasureSingle) DeleteObjects(ctx context.Context, bucket string, objec
 				DeleteMarker:          vr.Deleted,
 				DeleteMarkerVersionID: vr.VersionID,
 				DeleteMarkerMTime:     DeleteMarkerMTime{vr.ModTime},
-				ObjectName:            vr.Name,
+				ObjectName:            decodeDirObject(vr.Name),
 				ReplicationState:      vr.ReplicationState,
 			}
 		} else {
 			dobjects[i] = DeletedObject{
-				ObjectName:       vr.Name,
+				ObjectName:       decodeDirObject(vr.Name),
 				VersionID:        vr.VersionID,
 				ReplicationState: vr.ReplicationState,
 			}
@@ -2752,6 +2806,8 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, toObjectErr(err, bucket, object)
 	}
 
+	defer NSUpdated(bucket, object)
+
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -3040,6 +3096,33 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 	return nil
 }
 
+// Health - returns current status of the object layer health, for single drive
+// its as simple as returning healthy as long as drive is accessible.
+func (es *erasureSingle) Health(ctx context.Context, opts HealthOptions) HealthResult {
+	_, err := es.disk.DiskInfo(ctx)
+	if err != nil {
+		return HealthResult{}
+	}
+	if opts.Maintenance {
+		// Single drive cannot be put under maintenance.
+		return HealthResult{
+			Healthy:     false,
+			WriteQuorum: 1,
+		}
+	}
+	return HealthResult{
+		Healthy:     true,
+		WriteQuorum: 1,
+	}
+}
+
+// ReadHealth - returns current status of the object layer health for reads,
+// for single drive its as simple as returning healthy as long as drive is accessible.
+func (es *erasureSingle) ReadHealth(ctx context.Context) bool {
+	res := es.Health(ctx, HealthOptions{})
+	return res.Healthy
+}
+
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (es *erasureSingle) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
@@ -3235,7 +3318,7 @@ func (es *erasureSingle) NSScanner(ctx context.Context, bf *bloomFilter, updates
 	results := make([]dataUsageCache, 1)
 	var firstErr error
 
-	allBuckets, err := es.ListBuckets(ctx)
+	allBuckets, err := es.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}

cmd/erasure.go

@@ -288,7 +288,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 			disk := disks[i-1]
 
 			if disk == nil {
-				infos[i-1].Error = "nil disk"
+				infos[i-1].Error = "nil drive"
 				return
 			}
 
@@ -354,7 +354,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 	// Collect disks we can use.
 	disks, healing := er.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		logger.LogIf(ctx, errors.New("data-scanner: all disks are offline or being healed, skipping scanner cycle"))
+		logger.LogIf(ctx, errors.New("data-scanner: all drives are offline or being healed, skipping scanner cycle"))
 		return nil
 	}
 

cmd/erasure_test.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"crypto/rand"
 	"io"
-	"os"
 	"testing"
 )
 
@@ -118,20 +117,13 @@ type erasureTestSetup struct {
 	disks        []StorageAPI
 }
 
-// Removes the temporary disk directories.
-func (e erasureTestSetup) Remove() {
-	for _, path := range e.diskPaths {
-		os.RemoveAll(path)
-	}
-}
-
 // Returns an initialized setup for erasure tests.
-func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) {
+func newErasureTestSetup(tb testing.TB, dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) {
 	diskPaths := make([]string, dataBlocks+parityBlocks)
 	disks := make([]StorageAPI, len(diskPaths))
 	var err error
 	for i := range diskPaths {
-		disks[i], diskPaths[i], err = newXLStorageTestSetup()
+		disks[i], diskPaths[i], err = newXLStorageTestSetup(tb)
 		if err != nil {
 			return nil, err
 		}

cmd/format-disk-cache.go

@@ -110,7 +110,7 @@ func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
 // Creates a new cache format.json if unformatted.
 func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
 	// open file using READ & WRITE permission
-	file, err := os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o600)
+	file, err := os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o666)
 	if err != nil {
 		return err
 	}
@@ -155,7 +155,7 @@ func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bo
 	migrating := false
 	for i, drive := range drives {
 		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
+		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
 		if err != nil {
 			if osIsNotExist(err) {
 				continue
@@ -478,7 +478,7 @@ func migrateOldCache(ctx context.Context, c *diskCache) error {
 
 func migrateCacheFormatJSON(cacheFormatPath string) error {
 	// now migrate format.json
-	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
+	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
 	if err != nil {
 		return err
 	}

cmd/format-erasure.go

@@ -79,7 +79,7 @@ type formatErasureV1 struct {
 	formatMetaV1
 	Erasure struct {
 		Version string `json:"version"` // Version of 'xl' format.
-		Disk    string `json:"disk"`    // Disk field carries assigned disk uuid.
+		Disk    string `json:"drive"`   // Disk field carries assigned disk uuid.
 		// JBOD field carries the input disk order generated the first
 		// time when fresh disks were supplied.
 		JBOD []string `json:"jbod"`
@@ -199,7 +199,7 @@ func formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) {
 
 	version, err := formatGetBackendErasureVersion(formatData)
 	if err != nil {
-		return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
+		return nil, nil, fmt.Errorf("Drive %s: %w", export, err)
 	}
 
 	migrate := func(formatPath string, formatData []byte) ([]byte, fs.FileInfo, error) {
@@ -217,7 +217,7 @@ func formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) {
 	case formatErasureVersionV1:
 		formatData, err = formatErasureMigrateV1ToV2(formatData, version)
 		if err != nil {
-			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
+			return nil, nil, fmt.Errorf("Drive %s: %w", export, err)
 		}
 		// Migrate successful v1 => v2, proceed to v2 => v3
 		version = formatErasureVersionV2
@@ -225,7 +225,7 @@ func formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) {
 	case formatErasureVersionV2:
 		formatData, err = formatErasureMigrateV2ToV3(formatData, export, version)
 		if err != nil {
-			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
+			return nil, nil, fmt.Errorf("Drive %s: %w", export, err)
 		}
 		// Migrate successful v2 => v3, v3 is latest
 		// version = formatXLVersionV3
@@ -438,14 +438,14 @@ func checkFormatErasureValues(formats []*formatErasureV3, disks []StorageAPI, se
 			return err
 		}
 		if len(formats) != len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]) {
-			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",
+			return fmt.Errorf("%s drive is already being used in another erasure deployment. (Number of drives specified: %d but the number of drives found in the %s drive's format.json: %d)",
 				disks[i], len(formats), humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]))
 		}
 		// Only if custom erasure drive count is set, verify if the
 		// set_drive_count was manually set - we need to honor what is
 		// present on the drives.
 		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != setDriveCount {
-			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", disks[i], len(formatErasure.Erasure.Sets[0]), setDriveCount)
+			return fmt.Errorf("%s drive is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", disks[i], len(formatErasure.Erasure.Sets[0]), setDriveCount)
 		}
 	}
 	return nil
@@ -628,7 +628,7 @@ func formatErasureV3Check(reference *formatErasureV3, format *formatErasureV3) e
 			}
 		}
 	}
-	return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.Erasure.Sets)
+	return fmt.Errorf("DriveID %s not found in any drive sets %s", this, format.Erasure.Sets)
 }
 
 // saveFormatErasureAll - populates `format.json` on disks in its order.

cmd/format-erasure_test.go

@@ -104,11 +104,7 @@ func TestFormatErasureEmpty(t *testing.T) {
 // Tests xl format migration.
 func TestFormatErasureMigrate(t *testing.T) {
 	// Get test root.
-	rootPath, err := getTestRoot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	m := &formatErasureV1{}
 	m.Format = formatBackendErasure
@@ -153,13 +149,13 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 	if formatV3.Erasure.This != m.Erasure.Disk {
-		t.Fatalf("expected disk uuid: %s, got: %s", m.Erasure.Disk, formatV3.Erasure.This)
+		t.Fatalf("expected drive uuid: %s, got: %s", m.Erasure.Disk, formatV3.Erasure.This)
 	}
 	if len(formatV3.Erasure.Sets) != 1 {
 		t.Fatalf("expected single set after migrating from v1 to v3, but found %d", len(formatV3.Erasure.Sets))
 	}
 	if !reflect.DeepEqual(formatV3.Erasure.Sets[0], m.Erasure.JBOD) {
-		t.Fatalf("expected disk uuid: %v, got: %v", m.Erasure.JBOD, formatV3.Erasure.Sets[0])
+		t.Fatalf("expected drive uuid: %v, got: %v", m.Erasure.JBOD, formatV3.Erasure.Sets[0])
 	}
 
 	m = &formatErasureV1{}

cmd/format-fs.go

@@ -165,7 +165,7 @@ func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) e
 func createFormatFS(fsFormatPath string) error {
 	// Attempt a write lock on formatConfigFile `format.json`
 	// file stored in minioMetaBucket(.minio.sys) directory.
-	lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o600)
+	lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o666)
 	if err != nil {
 		return err
 	}
@@ -256,7 +256,7 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			// Hold write lock during migration so that we do not disturb any
 			// minio processes running in parallel.
 			var wlk *lock.LockedFile
-			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0o666)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again.
 				time.Sleep(100 * time.Millisecond)
@@ -357,7 +357,7 @@ func formatFSFixDeploymentID(ctx context.Context, fsFormatPath string) error {
 		case <-ctx.Done():
 			return fmt.Errorf("Initializing FS format stopped gracefully")
 		default:
-			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0o666)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again
 				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())

cmd/fs-v1-helpers_test.go

@@ -30,11 +30,10 @@ import (
 
 func TestFSRenameFile(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = fsMkdir(GlobalContext, pathJoin(path, "testvolume1")); err != nil {
 		t.Fatal(err)
@@ -55,11 +54,10 @@ func TestFSRenameFile(t *testing.T) {
 
 func TestFSStats(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 
@@ -183,11 +181,10 @@ func TestFSStats(t *testing.T) {
 
 func TestFSCreateAndOpen(t *testing.T) {
 	// Setup test environment.
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
@@ -248,11 +245,10 @@ func TestFSCreateAndOpen(t *testing.T) {
 
 func TestFSDeletes(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
@@ -350,11 +346,10 @@ func TestFSDeletes(t *testing.T) {
 
 func BenchmarkFSDeleteFile(b *testing.B) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(b)
 	if err != nil {
 		b.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(path, "benchmark")); err != nil {
@@ -384,11 +379,10 @@ func BenchmarkFSDeleteFile(b *testing.B) {
 // Tests fs removes.
 func TestFSRemoves(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
@@ -501,11 +495,10 @@ func TestFSRemoves(t *testing.T) {
 
 func TestFSRemoveMeta(t *testing.T) {
 	// create xlStorage test setup
-	_, fsPath, err := newXLStorageTestSetup()
+	_, fsPath, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(fsPath)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(fsPath, "success-vol")); err != nil {
@@ -529,10 +522,7 @@ func TestFSRemoveMeta(t *testing.T) {
 
 	defer rwPool.Close(filePath)
 
-	tmpDir, tmpErr := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if tmpErr != nil {
-		t.Fatal(tmpErr)
-	}
+	tmpDir := t.TempDir()
 
 	if err := fsRemoveMeta(GlobalContext, fsPath, filePath, tmpDir); err != nil {
 		t.Fatalf("Unable to remove file, %s", err)
@@ -548,15 +538,9 @@ func TestFSRemoveMeta(t *testing.T) {
 }
 
 func TestFSIsFile(t *testing.T) {
-	dirPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create tmp directory %s", err)
-	}
-	defer os.RemoveAll(dirPath)
+	filePath := pathJoin(t.TempDir(), "tmpfile")
 
-	filePath := pathJoin(dirPath, "tmpfile")
-
-	if err = ioutil.WriteFile(filePath, nil, 0o777); err != nil {
+	if err := ioutil.WriteFile(filePath, nil, 0o777); err != nil {
 		t.Fatalf("Unable to create file %s", filePath)
 	}
 

cmd/fs-v1-metadata_test.go

@@ -56,7 +56,7 @@ func TestReadFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	if _, err := obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
@@ -92,7 +92,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	if _, err := obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {

cmd/fs-v1-multipart.go

@@ -517,7 +517,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 		}
 	}
 
-	rc, _, err := fsOpenFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile), 0)
+	rc, _, err := fsOpenFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile), 0o666)
 	if err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
 			return result, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}

cmd/fs-v1-multipart_test.go

@@ -45,7 +45,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 
 	// Create a context we can cancel.
 	ctx, cancel := context.WithCancel(GlobalContext)
-	obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{})
+	obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})
 
 	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
 	if err != nil {
@@ -99,7 +99,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -124,7 +124,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	data := []byte("12345")
 	dataLen := int64(len(data))
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -157,7 +157,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -190,7 +190,7 @@ func TestCompleteMultipartUpload(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -227,7 +227,7 @@ func TestAbortMultipartUpload(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -258,7 +258,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 

cmd/fs-v1-rwpool.go

@@ -187,6 +187,8 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 			return nil, errFileAccessDenied
 		case isSysErrIsDir(err):
 			return nil, errIsNotRegular
+		case isSysErrNotDir(err):
+			return nil, errFileAccessDenied
 		case isSysErrPathNotFound(err):
 			return nil, errFileAccessDenied
 		default:

cmd/fs-v1-rwpool_test.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"os"
 	"runtime"
 	"testing"
 
@@ -48,11 +47,10 @@ func TestRWPoolLongPath(t *testing.T) {
 // Tests all RWPool methods.
 func TestRWPool(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	rwPool := &fsIOPool{
 		readersMap: make(map[string]*lock.RLockedFile),

cmd/fs-v1.go

@@ -254,7 +254,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 		return err
 	}
 	totalCache.Info.Name = dataUsageRoot
-	buckets, err := fs.ListBuckets(ctx)
+	buckets, err := fs.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -453,7 +453,7 @@ func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileI
 }
 
 // MakeBucketWithLocation - create a new bucket, returns if it already exists.
-func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	if opts.LockEnabled || opts.VersioningEnabled {
 		return NotImplemented{}
 	}
@@ -524,7 +524,7 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 }
 
 // GetBucketInfo - fetch bucket metadata info.
-func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
+func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
 	st, err := fs.statBucketDir(ctx, bucket)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
@@ -543,7 +543,7 @@ func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucke
 }
 
 // ListBuckets - list all s3 compatible buckets (directories) at fsPath.
-func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
+func (fs *FSObjects) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
 	if err := checkPathLength(fs.fsPath); err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
@@ -867,7 +867,7 @@ func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object s
 	// Read `fs.json` to perhaps contend with
 	// parallel Put() operations.
 
-	rc, _, err := fsOpenFile(ctx, fsMetaPath, 0)
+	rc, _, err := fsOpenFile(ctx, fsMetaPath, 0o666)
 	if err == nil {
 		fsMetaBuf, rerr := ioutil.ReadAll(rc)
 		rc.Close()

cmd/fs-v1_test.go

@@ -41,7 +41,7 @@ func TestNewFS(t *testing.T) {
 	}
 	_, err = NewFSObjectLayer(disk)
 	if err != nil {
-		errMsg := "Unable to recognize backend format, Disk is not in FS format."
+		errMsg := "Unable to recognize backend format, Drive is not in FS format."
 		if err.Error() == errMsg {
 			t.Errorf("Expecting %s, got %s", errMsg, err)
 		}
@@ -62,7 +62,7 @@ func TestFSShutdown(t *testing.T) {
 		fs := obj.(*FSObjects)
 
 		objectContent := "12345"
-		obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+		obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 		obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 		return fs, disk
 	}
@@ -95,18 +95,18 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucketWithLocation(GlobalContext, "a", BucketOptions{})
+	err := obj.MakeBucketWithLocation(GlobalContext, "a", MakeBucketOptions{})
 	if !isSameType(err, BucketNameInvalid{}) {
 		t.Fatal("BucketNameInvalid error not returned")
 	}
 
-	err = obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	err = obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Test with valid parameters
-	info, err := fs.GetBucketInfo(GlobalContext, bucketName)
+	info, err := fs.GetBucketInfo(GlobalContext, bucketName, BucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -115,7 +115,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	}
 
 	// Test with non-existent bucket
-	_, err = fs.GetBucketInfo(GlobalContext, "a")
+	_, err = fs.GetBucketInfo(GlobalContext, "a", BucketOptions{})
 	if !isSameType(err, BucketNotFound{}) {
 		t.Fatal("BucketNotFound error not returned")
 	}
@@ -123,7 +123,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	// Check for buckets and should get disk not found.
 	os.RemoveAll(disk)
 
-	if _, err = fs.GetBucketInfo(GlobalContext, bucketName); err != nil {
+	if _, err = fs.GetBucketInfo(GlobalContext, bucketName, BucketOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("BucketNotFound error not returned")
 		}
@@ -139,7 +139,7 @@ func TestFSPutObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "1/2/3/4/object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -179,7 +179,7 @@ func TestFSDeleteObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 	obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 
 	// Test with invalid bucket name
@@ -223,7 +223,7 @@ func TestFSDeleteBucket(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -242,7 +242,7 @@ func TestFSDeleteBucket(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 
 	// Delete bucket should get error disk not found.
 	os.RemoveAll(disk)
@@ -264,7 +264,7 @@ func TestFSListBuckets(t *testing.T) {
 	fs := obj.(*FSObjects)
 
 	bucketName := "bucket"
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -279,7 +279,7 @@ func TestFSListBuckets(t *testing.T) {
 	f.Close()
 
 	// Test list buckets to have only one entry.
-	buckets, err := fs.ListBuckets(GlobalContext)
+	buckets, err := fs.ListBuckets(GlobalContext, BucketOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -289,7 +289,7 @@ func TestFSListBuckets(t *testing.T) {
 
 	// Test ListBuckets with disk not found.
 	os.RemoveAll(disk)
-	if _, err := fs.ListBuckets(GlobalContext); err != nil {
+	if _, err := fs.ListBuckets(GlobalContext, BucketOptions{}); err != nil {
 		if err != errDiskNotFound {
 			t.Fatal("Unexpected error: ", err)
 		}

cmd/gateway-main.go

@@ -310,7 +310,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
 
 	if gatewayName == NASBackendGateway {
-		buckets, err := newObject.ListBuckets(GlobalContext)
+		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}
@@ -323,7 +323,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		// initialize the new disk cache objects.
 		var cacheAPI CacheObjectLayer
 		cacheAPI, err = newServerCacheObjects(GlobalContext, globalCacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
+		logger.FatalIf(err, "Unable to initialize drive caching")
 
 		globalObjLayerMutex.Lock()
 		globalCacheObjectAPI = cacheAPI
@@ -332,7 +332,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Populate existing buckets to the etcd backend
 	if globalDNSConfig != nil {
-		buckets, err := newObject.ListBuckets(GlobalContext)
+		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}

cmd/gateway/s3/gateway-s3-sse.go

@@ -734,7 +734,7 @@ func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, clea
 
 // cleanupStaleUploads removes old custom encryption multipart uploads on backend
 func (l *s3EncObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
-	buckets, err := l.s3Objects.ListBuckets(ctx)
+	buckets, err := l.s3Objects.ListBuckets(ctx, minio.BucketOptions{})
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return

cmd/gateway/s3/gateway-s3.go

@@ -325,7 +325,7 @@ func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []
 }
 
 // MakeBucket creates a new container on S3 backend.
-func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
+func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.MakeBucketOptions) error {
 	if opts.LockEnabled || opts.VersioningEnabled {
 		return minio.NotImplemented{}
 	}
@@ -348,7 +348,7 @@ func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, o
 }
 
 // GetBucketInfo gets bucket metadata..
-func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
+func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string, opts minio.BucketOptions) (bi minio.BucketInfo, e error) {
 	buckets, err := l.Client.ListBuckets(ctx)
 	if err != nil {
 		// Listbuckets may be disallowed, proceed to check if
@@ -381,7 +381,7 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.
 }
 
 // ListBuckets lists all S3 buckets
-func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
+func (l *s3Objects) ListBuckets(ctx context.Context, opts minio.BucketOptions) ([]minio.BucketInfo, error) {
 	buckets, err := l.Client.ListBuckets(ctx)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err)

cmd/global-heal.go

@@ -209,7 +209,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 
 		if serverDebugLog {
-			console.Debugf(color.Green("healDisk:")+" healing bucket %s content on %s erasure set\n",
+			console.Debugf(color.Green("healDrive:")+" healing bucket %s content on %s erasure set\n",
 				bucket, humanize.Ordinal(tracker.SetIndex+1))
 		}
 

cmd/iam-object-store.go

@@ -90,6 +90,9 @@ func (iamOS *IAMObjectStore) getUsersSysType() UsersSysType {
 // 3. Migrate user identity json file to include version info.
 func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context) error {
 	basePrefix := iamConfigUsersPrefix
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix) {
 		if item.Err != nil {
 			return item.Err
@@ -274,6 +277,8 @@ func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m
 }
 
 func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPoliciesPrefix) {
 		if item.Err != nil {
 			return item.Err
@@ -323,6 +328,8 @@ func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType
 		basePrefix = iamConfigUsersPrefix
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix) {
 		if item.Err != nil {
 			return item.Err
@@ -350,6 +357,8 @@ func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[
 }
 
 func (iamOS *IAMObjectStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigGroupsPrefix) {
 		if item.Err != nil {
 			return item.Err
@@ -392,6 +401,8 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 			basePath = iamConfigPolicyDBUsersPrefix
 		}
 	}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePath) {
 		if item.Err != nil {
 			return item.Err
@@ -432,7 +443,8 @@ var (
 
 func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
 	res := make(map[string][]string)
-
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) {
 		if item.Err != nil {
 			return nil, item.Err

cmd/iam-store.go

@@ -337,33 +337,16 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 		return c.iamGroupPolicyMap[name].toSlice(), c.iamGroupPolicyMap[name].UpdatedAt, nil
 	}
 
-	if name == globalActiveCred.AccessKey {
-		return []string{"consoleAdmin"}, time.Time{}, nil
-	}
-
 	// When looking for a user's policies, we also check if the user
 	// and the groups they are member of are enabled.
-	var parentName string
 	u, ok := c.iamUsersMap[name]
 	if ok {
 		if !u.Credentials.IsValid() {
 			return nil, time.Time{}, nil
 		}
-		parentName = u.Credentials.ParentUser
 	}
 
-	mp, ok := c.iamUserPolicyMap[name]
-	if !ok {
-		// Service accounts with root credentials, inherit parent permissions
-		if parentName == globalActiveCred.AccessKey && u.Credentials.IsServiceAccount() {
-			// even if this is set, the claims present in the service
-			// accounts apply the final permissions if any.
-			return []string{"consoleAdmin"}, mp.UpdatedAt, nil
-		}
-		if parentName != "" {
-			mp = c.iamUserPolicyMap[parentName]
-		}
-	}
+	mp := c.iamUserPolicyMap[name]
 
 	// returned policy could be empty
 	policies := mp.toSlice()

cmd/iam.go

@@ -1458,7 +1458,7 @@ const sessionPolicyNameExtracted = iampolicy.SessionPolicyName + "-extracted"
 // IsAllowedServiceAccount - checks if the given service account is allowed to perform
 // actions. The permission of the parent user is checked first
 func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser string) bool {
-	// Now check if we have a subject claim
+	// Verify if the parent claim matches the parentUser.
 	p, ok := args.Claims[parentClaim]
 	if ok {
 		parentInClaim, ok := p.(string)
@@ -1478,41 +1478,56 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 		return false
 	}
 
-	// Check policy for parent user of service account.
-	svcPolicies, err := sys.PolicyDBGet(parentUser, false, args.Groups...)
-	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return false
-	}
+	isOwnerDerived := parentUser == globalActiveCred.AccessKey
 
-	if len(svcPolicies) == 0 {
-		// If parent user has no policies, check for OpenID
-		// claims/RoleARN in case it exists.
-		roleArn := args.GetRoleArn()
-		if roleArn != "" {
-			arn, err := arn.Parse(roleArn)
-			if err != nil {
-				logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
-				return false
-			}
-			svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
-		} else {
-			// If there is no roleArn claim, check the OpenID
-			// provider's policy claim.
+	var err error
+	var svcPolicies []string
+	roleArn := args.GetRoleArn()
+
+	switch {
+	case isOwnerDerived:
+		// All actions are allowed by default and no policy evaluation is
+		// required.
+
+	case roleArn != "":
+		arn, err := arn.Parse(roleArn)
+		if err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
+			return false
+		}
+		svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
+
+	default:
+		// Check policy for parent user of service account.
+		svcPolicies, err = sys.PolicyDBGet(parentUser, false, args.Groups...)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			return false
+		}
+
+		// Finally, if there is no parent policy, check if a policy claim is
+		// present.
+		if len(svcPolicies) == 0 {
 			policySet, _ := iampolicy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID())
 			svcPolicies = policySet.ToSlice()
 		}
-		if len(svcPolicies) == 0 {
-			return false
-		}
 	}
 
-	// Policies were found, evaluate all of them.
-	availablePoliciesStr, combinedPolicy := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
-	if availablePoliciesStr == "" {
+	// Defensive code: Do not allow any operation if no policy is found.
+	if !isOwnerDerived && len(svcPolicies) == 0 {
 		return false
 	}
 
+	var combinedPolicy iampolicy.Policy
+	// Policies were found, evaluate all of them.
+	if !isOwnerDerived {
+		availablePoliciesStr, c := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
+		if availablePoliciesStr == "" {
+			return false
+		}
+		combinedPolicy = c
+	}
+
 	parentArgs := args
 	parentArgs.AccountName = parentUser
 	// These are dynamic values set them appropriately.
@@ -1532,7 +1547,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 	}
 
 	if saPolicyClaimStr == inheritedPolicyType {
-		return combinedPolicy.IsAllowed(parentArgs)
+		return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)
 	}
 
 	// Now check if we have a sessionPolicy.
@@ -1558,37 +1573,51 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 
 	// This can only happen if policy was set but with an empty JSON.
 	if subPolicy.Version == "" && len(subPolicy.Statements) == 0 {
-		return combinedPolicy.IsAllowed(parentArgs)
+		return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)
 	}
 
 	if subPolicy.Version == "" {
 		return false
 	}
 
-	return combinedPolicy.IsAllowed(parentArgs) && subPolicy.IsAllowed(parentArgs)
+	return subPolicy.IsAllowed(parentArgs) && (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs))
 }
 
 // IsAllowedSTS is meant for STS based temporary credentials,
 // which implements claims validation and verification other than
 // applying policies.
 func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
+	// 1. Determine mapped policies
+
+	isOwnerDerived := parentUser == globalActiveCred.AccessKey
 	var policies []string
 	roleArn := args.GetRoleArn()
-	if roleArn != "" {
+
+	switch {
+	case isOwnerDerived:
+		// All actions are allowed by default and no policy evaluation is
+		// required.
+
+	case roleArn != "":
+		// If a roleARN is present, the role policy is applied.
 		arn, err := arn.Parse(roleArn)
 		if err != nil {
 			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
 			return false
 		}
 		policies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
-	} else {
-		// Lookup the parent user's mapping if there's no role-ARN.
+
+	default:
+		// Otherwise, inherit parent user's policy
 		var err error
 		policies, err = sys.store.PolicyDBGet(parentUser, false, args.Groups...)
 		if err != nil {
 			logger.LogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err))
 			return false
 		}
+
+		// Finally, if there is no parent policy, check if a policy claim is
+		// present in the session token.
 		if len(policies) == 0 {
 			// If there is no parent policy mapping, we fall back to
 			// using policy claim from JWT.
@@ -1599,27 +1628,37 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 			}
 			policies = policySet.ToSlice()
 		}
+
 	}
 
 	// Defensive code: Do not allow any operation if no policy is found in the session token
-	if len(policies) == 0 {
+	if !isOwnerDerived && len(policies) == 0 {
 		return false
 	}
 
-	combinedPolicy, err := sys.store.GetPolicy(strings.Join(policies, ","))
-	if err == errNoSuchPolicy {
-		for _, pname := range policies {
-			_, err := sys.store.GetPolicy(pname)
-			if err == errNoSuchPolicy {
-				// all policies presented in the claim should exist
-				logger.LogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
-				return false
+	// 2. Combine the mapped policies into a single combined policy.
+
+	var combinedPolicy iampolicy.Policy
+	if !isOwnerDerived {
+		var err error
+		combinedPolicy, err = sys.store.GetPolicy(strings.Join(policies, ","))
+		if err == errNoSuchPolicy {
+			for _, pname := range policies {
+				_, err := sys.store.GetPolicy(pname)
+				if err == errNoSuchPolicy {
+					// all policies presented in the claim should exist
+					logger.LogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
+					return false
+				}
 			}
+			logger.LogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
+			return false
 		}
-		logger.LogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
-		return false
+
 	}
 
+	// 3. If an inline session-policy is present, evaluate it.
+
 	// These are dynamic values set them appropriately.
 	args.ConditionValues["username"] = []string{parentUser}
 	args.ConditionValues["userid"] = []string{parentUser}
@@ -1627,12 +1666,12 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 	// Now check if we have a sessionPolicy.
 	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args)
 	if hasSessionPolicy {
-		return isAllowedSP && combinedPolicy.IsAllowed(args)
+		return isAllowedSP && (isOwnerDerived || combinedPolicy.IsAllowed(args))
 	}
 
 	// Sub policy not set, this is most common since subPolicy
 	// is optional, use the inherited policies.
-	return combinedPolicy.IsAllowed(args)
+	return isOwnerDerived || combinedPolicy.IsAllowed(args)
 }
 
 func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAllowed bool) {

cmd/listen-notification-handlers.go

@@ -113,7 +113,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	}
 
 	if bucketName != "" {
-		if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
+		if _, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{}); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}

cmd/metacache-set.go

@@ -1332,7 +1332,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				if err != nil {
 					if disks[i] != nil {
 						combinedErr = append(combinedErr,
-							fmt.Sprintf("disk %s returned: %s", disks[i], err))
+							fmt.Sprintf("drive %s returned: %s", disks[i], err))
 					} else {
 						combinedErr = append(combinedErr, err.Error())
 					}

cmd/metrics-v2.go

@@ -332,7 +332,7 @@ func getNodeDiskAPILatencyMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      apiLatencyMicroSec,
-		Help:      "Average last minute latency in µs for disk API storage operations",
+		Help:      "Average last minute latency in µs for drive API storage operations",
 		Type:      gaugeMetric,
 	}
 }
@@ -342,7 +342,7 @@ func getNodeDiskUsedBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      usedBytes,
-		Help:      "Total storage used on a disk",
+		Help:      "Total storage used on a drive",
 		Type:      gaugeMetric,
 	}
 }
@@ -352,7 +352,7 @@ func getNodeDiskFreeBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      freeBytes,
-		Help:      "Total storage available on a disk",
+		Help:      "Total storage available on a drive",
 		Type:      gaugeMetric,
 	}
 }
@@ -362,7 +362,7 @@ func getClusterDisksOfflineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      offlineTotal,
-		Help:      "Total disks offline",
+		Help:      "Total drives offline",
 		Type:      gaugeMetric,
 	}
 }
@@ -372,7 +372,7 @@ func getClusterDisksOnlineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      onlineTotal,
-		Help:      "Total disks online",
+		Help:      "Total drives online",
 		Type:      gaugeMetric,
 	}
 }
@@ -382,7 +382,7 @@ func getClusterDisksTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      total,
-		Help:      "Total disks",
+		Help:      "Total drives",
 		Type:      gaugeMetric,
 	}
 }
@@ -402,7 +402,7 @@ func getNodeDiskTotalBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      totalBytes,
-		Help:      "Total storage on a disk",
+		Help:      "Total storage on a drive",
 		Type:      gaugeMetric,
 	}
 }
@@ -702,7 +702,7 @@ func getCacheHitsTotalMD() MetricDescription {
 		Namespace: minioNamespace,
 		Subsystem: cacheSubsystem,
 		Name:      hitsTotal,
-		Help:      "Total number of disk cache hits",
+		Help:      "Total number of drive cache hits",
 		Type:      counterMetric,
 	}
 }
@@ -712,7 +712,7 @@ func getCacheHitsMissedTotalMD() MetricDescription {
 		Namespace: minioNamespace,
 		Subsystem: cacheSubsystem,
 		Name:      missedTotal,
-		Help:      "Total number of disk cache misses",
+		Help:      "Total number of drive cache misses",
 		Type:      counterMetric,
 	}
 }
@@ -752,7 +752,7 @@ func getCacheTotalBytesMD() MetricDescription {
 		Namespace: minioNamespace,
 		Subsystem: cacheSubsystem,
 		Name:      totalBytes,
-		Help:      "Total size of cache disk in bytes",
+		Help:      "Total size of cache drive in bytes",
 		Type:      gaugeMetric,
 	}
 }
@@ -996,7 +996,7 @@ func getMinioProcMetrics() *MetricsGroup {
 		metrics = make([]Metric, 0, 20)
 		p, err := procfs.Self()
 		if err != nil {
-			logger.LogOnceIf(ctx, err, nodeMetricNamespace)
+			logger.LogOnceIf(ctx, err, string(nodeMetricNamespace))
 			return
 		}
 

cmd/metrics.go

@@ -273,7 +273,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
 			prometheus.BuildFQName(cacheNamespace, "hits", "total"),
-			"Total number of disk cache hits in current MinIO instance",
+			"Total number of drive cache hits in current MinIO instance",
 			nil, nil),
 		prometheus.CounterValue,
 		float64(cacheObjLayer.CacheStats().getHits()),
@@ -281,7 +281,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
 			prometheus.BuildFQName(cacheNamespace, "misses", "total"),
-			"Total number of disk cache misses in current MinIO instance",
+			"Total number of drive cache misses in current MinIO instance",
 			nil, nil),
 		prometheus.CounterValue,
 		float64(cacheObjLayer.CacheStats().getMisses()),
@@ -328,7 +328,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName("cache", "total", "size"),
-				"Indicates total size of cache disk",
+				"Indicates total size of cache drive",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
 			float64(cdStats.TotalCapacity),
@@ -593,7 +593,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
 			prometheus.BuildFQName(minioNamespace, "disks", "offline"),
-			"Total number of offline disks in current MinIO server instance",
+			"Total number of offline drives in current MinIO server instance",
 			nil, nil),
 		prometheus.GaugeValue,
 		float64(offlineDisks.Sum()),
@@ -602,8 +602,8 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	// MinIO Total Disks per node
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName(minioNamespace, "disks", "total"),
-			"Total number of disks for current MinIO server instance",
+			prometheus.BuildFQName(minioNamespace, "drives", "total"),
+			"Total number of drives for current MinIO server instance",
 			nil, nil),
 		prometheus.GaugeValue,
 		float64(totalDisks.Sum()),
@@ -614,7 +614,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName(diskNamespace, "storage", "used"),
-				"Total disk storage used on the disk",
+				"Total disk storage used on the drive",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
 			float64(disk.UsedSpace),
@@ -625,7 +625,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName(diskNamespace, "storage", "available"),
-				"Total available space left on the disk",
+				"Total available space left on the drive",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
 			float64(disk.AvailableSpace),
@@ -636,7 +636,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
 				prometheus.BuildFQName(diskNamespace, "storage", "total"),
-				"Total space on the disk",
+				"Total space on the drive",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
 			float64(disk.TotalSpace),

cmd/notification.go

@@ -297,9 +297,7 @@ func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerEr
 }
 
 // DownloadProfilingData - download profiling data from all remote peers.
-func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) bool {
-	profilingDataFound := false
-
+func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) (profilingDataFound bool) {
 	// Initialize a zip writer which will provide a zipped content
 	// of profiling data of all nodes
 	zipWriter := zip.NewWriter(writer)
@@ -320,34 +318,11 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		profilingDataFound = true
 
 		for typ, data := range data {
-			// Send profiling data to zip as file
-			header, zerr := zip.FileInfoHeader(dummyFileInfo{
-				name:    fmt.Sprintf("profile-%s-%s", client.host.String(), typ),
-				size:    int64(len(data)),
-				mode:    0o600,
-				modTime: UTCNow(),
-				isDir:   false,
-				sys:     nil,
-			})
-			if zerr != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, zerr)
-				continue
-			}
-			header.Method = zip.Deflate
-			zwriter, zerr := zipWriter.CreateHeader(header)
-			if zerr != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, zerr)
-				continue
-			}
-			if _, err = io.Copy(zwriter, bytes.NewReader(data)); err != nil {
+			err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data)
+			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 				ctx := logger.SetReqInfo(ctx, reqInfo)
 				logger.LogIf(ctx, err)
-				continue
 			}
 		}
 	}
@@ -371,34 +346,18 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 
 	// Send profiling data to zip as file
 	for typ, data := range data {
-		header, zerr := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profile-%s-%s", thisAddr, typ),
-			size:    int64(len(data)),
-			mode:    0o600,
-			modTime: UTCNow(),
-			isDir:   false,
-			sys:     nil,
-		})
-		if zerr != nil {
-			return profilingDataFound
-		}
-		header.Method = zip.Deflate
-
-		zwriter, zerr := zipWriter.CreateHeader(header)
-		if zerr != nil {
-			return profilingDataFound
-		}
-
-		if _, err = io.Copy(zwriter, bytes.NewReader(data)); err != nil {
-			return profilingDataFound
+		err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data)
+		if err != nil {
+			logger.LogIf(ctx, err)
 		}
 	}
 
-	return profilingDataFound
+	appendClusterMetaInfoToZip(ctx, zipWriter)
+	return
 }
 
-// ServerUpdate - updates remote peers.
-func (sys *NotificationSys) ServerUpdate(ctx context.Context, u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string) []NotificationPeerErr {
+// VerifyBinary - asks remote peers to verify the checksum
+func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, reader []byte) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
@@ -406,7 +365,22 @@ func (sys *NotificationSys) ServerUpdate(ctx context.Context, u *url.URL, sha256
 		}
 		client := client
 		ng.Go(ctx, func() error {
-			return client.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo)
+			return client.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
+// CommitBinary - asks remote peers to overwrite the old binary with the new one
+func (sys *NotificationSys) CommitBinary(ctx context.Context) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(ctx, func() error {
+			return client.CommitBinary(ctx)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -475,14 +449,14 @@ func (sys *NotificationSys) updateBloomFilter(ctx context.Context, current uint6
 			serverBF, err := client.cycleServerBloomFilter(ctx, req)
 			if false && intDataUpdateTracker.debug {
 				b, _ := json.MarshalIndent(serverBF, "", "  ")
-				logger.Info("Disk %v, Bloom filter: %v", client.host.Name, string(b))
+				logger.Info("Drive %v, Bloom filter: %v", client.host.Name, string(b))
 			}
 			// Keep lock while checking result.
 			mu.Lock()
 			defer mu.Unlock()
 
 			if err != nil || !serverBF.Complete || bf == nil {
-				logger.LogOnceIf(ctx, err, fmt.Sprintf("host:%s, cycle:%d", client.host, current), client.cycleServerBloomFilter)
+				logger.LogOnceIf(ctx, err, client.host.String(), client.cycleServerBloomFilter)
 				bf = nil
 				return nil
 			}
@@ -743,7 +717,7 @@ func (sys *NotificationSys) InitBucketTargets(ctx context.Context, objAPI Object
 			if res.Err != nil {
 				reqInfo := &logger.ReqInfo{}
 				reqInfo.AppendTags("targetID", res.ID.Name)
-				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID)
+				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
 			}
 		}
 	}()

cmd/object-api-common.go

@@ -42,6 +42,9 @@ const (
 	// Buckets meta prefix.
 	bucketMetaPrefix = "buckets"
 
+	// Deleted Buckets prefix.
+	deletedBucketsPrefix = ".deleted"
+
 	// ETag (hex encoded md5sum) of empty string.
 	emptyETag = "d41d8cd98f00b204e9800998ecf8427e"
 )

cmd/object-api-datatypes.go

@@ -78,6 +78,7 @@ type BucketInfo struct {
 
 	// Date and time when the bucket was created.
 	Created time.Time
+	Deleted time.Time
 }
 
 // ObjectInfo - represents object metadata.

cmd/object-api-deleteobject_test.go

@@ -84,7 +84,7 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	for i, testCase := range testCases {
-		err := obj.MakeBucketWithLocation(context.Background(), testCase.bucketName, BucketOptions{})
+		err := obj.MakeBucketWithLocation(context.Background(), testCase.bucketName, MakeBucketOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}

cmd/object-api-errors.go

@@ -186,7 +186,7 @@ func (e SignatureDoesNotMatch) Error() string {
 type StorageFull struct{}
 
 func (e StorageFull) Error() string {
-	return "Storage reached its minimum free disk threshold."
+	return "Storage reached its minimum free drive threshold."
 }
 
 // SlowDown  too many file descriptors open or backend busy .

cmd/object-api-getobjectinfo_test.go

@@ -31,7 +31,7 @@ func TestGetObjectInfo(t *testing.T) {
 // Testing GetObjectInfo().
 func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// This bucket is used for testing getObjectInfo operations.
-	err := obj.MakeBucketWithLocation(context.Background(), "test-getobjectinfo", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "test-getobjectinfo", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}

cmd/object-api-input-checks.go

@@ -198,12 +198,12 @@ func checkPutObjectArgs(ctx context.Context, bucket, object string, obj getBucke
 }
 
 type getBucketInfoI interface {
-	GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
+	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error)
 }
 
 // Checks whether bucket exists and returns appropriate error if not.
 func checkBucketExist(ctx context.Context, bucket string, obj getBucketInfoI) error {
-	_, err := obj.GetBucketInfo(ctx, bucket)
+	_, err := obj.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		return err
 	}

cmd/object-api-interface.go

@@ -102,18 +102,25 @@ type TransitionOptions struct {
 	ExpireRestored bool
 }
 
-// BucketOptions represents bucket options for ObjectLayer bucket operations
-type BucketOptions struct {
+// MakeBucketOptions represents bucket options for ObjectLayer bucket operations
+type MakeBucketOptions struct {
 	Location          string
 	LockEnabled       bool
 	VersioningEnabled bool
-	ForceCreate       bool // Create buckets even if they are already created.
+	ForceCreate       bool      // Create buckets even if they are already created.
+	CreatedAt         time.Time // only for site replication
 }
 
 // DeleteBucketOptions provides options for DeleteBucket calls.
 type DeleteBucketOptions struct {
-	Force      bool // Force deletion
-	NoRecreate bool // Do not recreate on delete failures
+	Force      bool             // Force deletion
+	NoRecreate bool             // Do not recreate on delete failures
+	SRDeleteOp SRBucketDeleteOp // only when site replication is enabled
+}
+
+// BucketOptions provides options for ListBuckets and GetBucketInfo call.
+type BucketOptions struct {
+	Deleted bool // true only when site replication is enabled
 }
 
 // SetReplicaStatus sets replica status and timestamp for delete operations in ObjectOptions
@@ -187,9 +194,9 @@ type ObjectLayer interface {
 	LocalStorageInfo(ctx context.Context) (StorageInfo, []error)
 
 	// Bucket operations.
-	MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error
-	GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
+	MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error
+	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error)
+	ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error)
 	DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error
 	ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)