allow tagging policy condition for GetObject (#15777)

this is to ensure keep k8s pods running, when they reach a "crashloop" stage

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,5 +15,6 @@
 
 ## Checklist:
 - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
-- [ ] Documentation updated
 - [ ] Unit tests added/updated
+- [ ] Internal documentation updated
+- [ ] Create a documentation update request [here](https://github.com/minio/docs/issues/new?label=doc-change,title=Doc+Updated+Needed+For+PR+github.com%2fminio%2fminio%2fpull%2fNNNNN)

.github/workflows/depsreview.yaml

@@ -0,0 +1,14 @@
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v1

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2

.github/workflows/go-fips.yml

@@ -0,0 +1,51 @@
+name: FIPS Build Test
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Go BoringCrypto ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.18.5b7]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Setup dockerfile for build test
+        run: |
+          echo "FROM us-docker.pkg.dev/google.com/api-project-999119582588/go-boringcrypto/golang:${{ matrix.go-version }}" > Dockerfile.fips.test
+          echo "COPY . /minio" >> Dockerfile.fips.test
+          echo "WORKDIR /minio" >> Dockerfile.fips.test
+          echo "RUN make" >> Dockerfile.fips.test
+
+      - name: Build
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile.fips.test
+          push: false
+          load: true
+          tags: minio/fips-test:latest
+
+      # This should fail if grep returns non-zero exit
+      - name: Test binary
+        run: |
+          docker run --rm minio/fips-test:latest ./minio --version
+          docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio' | grep -q FIPS

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
@@ -48,3 +48,5 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify-healing
           make verify-healing-inconsistent-versions
+          make verify-healing-with-root-disks
+          make verify-healing-with-rewrite

.github/workflows/go-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
@@ -41,10 +41,7 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
-          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
-          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
-          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
           MINIO_KMS_AUTO_ENCRYPTION: on
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -108,6 +108,17 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
+      - name: Test with Access Management Plugin enabled
+        env:
+          LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          ETCD_SERVER: ${{ matrix.etcd }}
+          OPENID_TEST_SERVER: ${{ matrix.openid }}
+          POLICY_PLUGIN_ENDPOINT: "http://127.0.0.1:8080"
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          go run docs/iam/access-manager-plugin.go &
+          make test-iam
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |

.github/workflows/markdown-lint.yaml

@@ -27,4 +27,4 @@ jobs:
         npm install -g markdownlint-cli
         markdownlint --fix '**/*.md' \
           --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
-          --disable MD013 MD040
+          --disable MD013 MD040 MD051

.github/workflows/replication.yaml

@@ -1,4 +1,4 @@
-name: Multi-site replication tests
+name: MinIO advanced tests
 
 on:
   pull_request:
@@ -16,12 +16,12 @@ permissions:
 
 jobs:
   replication-test:
-    name: Replication Tests with Go ${{ matrix.go-version }}
+    name: Advanced Tests with Go ${{ matrix.go-version }}
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
 
     steps:
       - uses: actions/checkout@v2
@@ -37,11 +37,18 @@ jobs:
           key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Test Decom
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-decom
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-replication
+
       - name: Test MinIO IDP for automatic site replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/vulncheck.yml

@@ -0,0 +1,26 @@
+name: VulnCheck
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+jobs:
+  vulncheck:
+    name: Analysis
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+          check-latest: true
+      - name: Get official govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+        shell: bash
+      - name: Run govulncheck
+        run: govulncheck ./...
+        shell: bash

.gitignore

@@ -9,8 +9,7 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-vendor/**/*.js
-vendor/**/*.json
+vendor/
 .DS_Store
 *.syso
 coverage.txt
@@ -32,4 +31,10 @@ hash-set
 minio.RELEASE*
 mc
 nancy
-inspects/*
+inspects/*
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect

.golangci.yml

@@ -2,6 +2,13 @@ linters-settings:
   golint:
     min-confidence: 0
 
+  gofumpt:
+    lang-version: "1.18"
+
+    # Choose whether or not to use the extra rules that are disabled
+    # by default
+    extra-rules: false
+
   misspell:
     locale: US
 
@@ -14,28 +21,15 @@ linters:
     - govet
     - revive
     - ineffassign
-    - gosimple
-    - deadcode
-    - structcheck
     - gomodguard
     - gofmt
-    - unused
-    - structcheck
     - unconvert
-    - varcheck
+    - unused
     - gocritic
     - gofumpt
     - tenv
     - durationcheck
 
-linters-settings:
-  gofumpt:
-    lang-version: "1.17"
-
-    # Choose whether or not to use the extra rules that are disabled
-    # by default
-    extra-rules: false
-
 issues:
   exclude-use-default: false
   exclude:

.nancy-ignore

@@ -2,3 +2,5 @@ CVE-2020-26160
 CVE-2020-15136
 CVE-2020-15115
 CVE-2020-15114
+CVE-2022-2835
+CVE-2022-2837

Dockerfile.hotfix

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG RELEASE
 

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG TARGETARCH
 

Dockerfile.release.fips

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG TARGETARCH
 

Makefile

@@ -15,12 +15,12 @@ checks: ## check dependencies
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
 help: ## print this help
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}'
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@f3635b96e4838a6c773babb65ef35297fe5fe2f9
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -39,7 +39,14 @@ lint: ## runs golangci-lint suite of linters
 check: test
 test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@CGO_ENABLED=0 go test -tags kqueue ./...
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
+
+test-decom: install
+	@echo "Running minio decom tests"
+	@env bash $(PWD)/docs/distributed/decom.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
+	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
 test-upgrade: build
 	@echo "Running minio upgrade tests"
@@ -51,13 +58,14 @@ test-race: verifiers build ## builds minio, runs linters, tests (race)
 
 test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
 test-replication: install ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
+	@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
 
 test-site-replication-ldap: install ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with LDAP)"
@@ -73,18 +81,29 @@ test-site-replication-minio: install ## verify automatic site replication
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"
-	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
-	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
+	@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
+
+verify-healing-with-root-disks: ## verify healing root disks
+	@echo "Verify healing with root drives"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
+
+verify-healing-with-rewrite: ## verify healing to rewrite old xl.meta -> new xl.meta
+	@echo "Verify healing with rewrite"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/rewrite-old-new.sh)
 
 verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
-	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
 
 build: checks ## builds minio to $(PWD)
@@ -127,6 +146,8 @@ clean: ## cleanup all generated assets
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@find . -name '*~' | xargs rm -fv
+	@find . -name '.#*#' | xargs rm -fv
+	@find . -name '#*#' | xargs rm -fv
 	@rm -rvf minio
 	@rm -rvf build
 	@rm -rvf release

README.fips.md

@@ -0,0 +1,7 @@
+# MinIO FIPS Builds
+
+MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring).
+
+MinIO FIPS executables are available at <http://dl.min.io> - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
+
+We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes.

README.md

@@ -14,7 +14,7 @@ Use the following commands to run a standalone MinIO server as a container.
 
 Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
 require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
-with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
 for more complete documentation.
 
 ### Stable
@@ -32,7 +32,7 @@ root credentials. You can use the Browser to create buckets, upload objects, and
 
 You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
 [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
-see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
 > NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
 
@@ -40,7 +40,7 @@ see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to vi
 
 Use the following commands to run a standalone MinIO server on macOS.
 
-Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
 
 ### Homebrew (recommended)
 
@@ -60,7 +60,7 @@ brew install minio/stable/minio
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html/> to view MinIO SDKs for supported languages.
 
 ### Binary Download
 
@@ -74,7 +74,7 @@ chmod +x minio
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
 ## GNU/Linux
 
@@ -99,9 +99,9 @@ The following table lists supported architectures. Replace the `wget` URL with t
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
 
 ## Microsoft Windows
 
@@ -119,23 +119,23 @@ minio.exe server D:\
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
 
 ```sh
-GO111MODULE=on go install github.com/minio/minio@latest
+go install github.com/minio/minio@latest
 ```
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
 
-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
 
-> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
 
 MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
 
@@ -196,12 +196,6 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
 service iptables restart
 ```
 
-## Pre-existing data
-
-When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
-
-The above statement is also valid for all gateway backends.
-
 ## Test MinIO Connectivity
 
 ### Test using MinIO Console
@@ -228,7 +222,7 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
 
 ## Test using MinIO Client `mc`
 
-`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
+`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart) for further instructions.
 
 ## Upgrading MinIO
 
@@ -236,32 +230,30 @@ Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all tr
 
 > NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
 
-- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
+- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://min.io/docs/minio/linux/reference/minio-mc-admin/mc-admin-update.html)
 
 ```sh
 mc admin update <minio alias, e.g., myminio>
 ```
 
-- For deployments without external internet access (e.g. airgapped environments), download the binary from <https://dl.min.io> and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.
+- For deployments without external internet access (e.g. airgapped environments), download the binary from <https://dl.min.io> and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and proceed to perform `mc admin service restart alias/`.
 
-- For RPM/DEB installations, upgrade packages **parallelly** on all servers. Once upgraded, perform `systemctl restart minio` across all nodes in **parallel**. RPM/DEB based installations are usually automated using [`ansible`](https://github.com/minio/ansible-minio).
+- For installations using Systemd MinIO service, upgrade via RPM/DEB packages **parallelly** on all servers or replace the binary lets say `/opt/bin/minio` on all nodes, apply executable permissions `chmod +x /opt/bin/minio` and process to perform `mc admin service restart alias/`.
 
 ### Upgrade Checklist
 
 - Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
-- Read the release notes for the targeted MinIO release *before* performing any installation, there is no forced requirement to upgrade to latest releases every week. If it has a bug fix you are looking for then yes, else avoid actively upgrading a running production system.
-- Make sure MinIO process has write access to `/opt/bin` if you plan to use `mc admin update`. This is needed for MinIO to download the latest binary from <https://dl.min.io> and save it locally for upgrades.
-- `mc admin update` is not supported in kubernetes/container environments, container environments provide their own mechanisms for container updates.
+- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest releases upon every releases. Some releases may not be relevant to your setup, avoid upgrading production environments unnecessarily.
+- If you plan to use `mc admin update`, MinIO process must have write access to the parent directory where the binary is present on the host system.
+- `mc admin update` is not supported and should be avoided in kubernetes/container environments, please upgrade containers by upgrading relevant container images.
 - **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
 
 ## Explore Further
 
-- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
-- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
-- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
-- [Use `s3cmd` with MinIO Server](https://docs.min.io/docs/s3cmd-with-minio)
-- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
-- [The MinIO documentation website](https://docs.min.io)
+- [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
+- [Use `mc` with MinIO Server](https://min.io/docs/minio/linux/reference/minio-mc.html)
+- [Use `minio-go` SDK with MinIO Server](https://min.io/docs/minio/linux/developers/go/minio-go.html)
+- [The MinIO documentation website](https://min.io/docs/minio/linux/index.html)
 
 ## Contribute to MinIO Project
 

buildscripts/gen-ldflags.go

@@ -24,14 +24,18 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 )
 
 func genLDFlags(version string) string {
+	releaseTag, date := releaseTag(version)
+	copyrightYear := strconv.Itoa(date.Year())
 	ldflagsStr := "-s -w"
 	ldflagsStr += " -X github.com/minio/minio/cmd.Version=" + version
-	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
+	ldflagsStr += " -X github.com/minio/minio/cmd.CopyrightYear=" + copyrightYear
+	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
@@ -40,7 +44,7 @@ func genLDFlags(version string) string {
 }
 
 // genReleaseTag prints release tag to the console for easy git tagging.
-func releaseTag(version string) string {
+func releaseTag(version string) (string, time.Time) {
 	relPrefix := "DEVELOPMENT"
 	if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" {
 		relPrefix = prefix
@@ -53,14 +57,17 @@ func releaseTag(version string) string {
 
 	relTag := strings.Replace(version, " ", "-", -1)
 	relTag = strings.Replace(relTag, ":", "-", -1)
+	t, err := time.Parse("2006-01-02T15-04-05Z", relTag)
+	if err != nil {
+		panic(err)
+	}
 	relTag = strings.Replace(relTag, ",", "", -1)
 	relTag = relPrefix + "." + relTag
-
 	if relSuffix != "" {
 		relTag += "." + relSuffix
 	}
 
-	return relTag
+	return relTag, t
 }
 
 // commitID returns the abbreviated commit-id hash of the last commit.

buildscripts/heal-inconsistent-versions.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+function start_minio_4drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export MINIO_CI_CD=1
+
+    mkdir ${WORK_DIR}
+    C_PWD=${PWD}
+    if [ ! -x "$PWD/mc" ]; then
+	MC_BUILD_DIR="mc-$RANDOM"
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	    echo "failed to download https://github.com/minio/mc"
+	    purge "${MC_BUILD_DIR}"
+	    exit 1
+	fi
+
+	(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+	# remove mc source.
+	purge "${MC_BUILD_DIR}"
+    fi
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 5
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${PWD}/mc" mb --with-versioning minio/bucket
+
+    for i in $(seq 1 4); do
+	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+
+	sudo chown -R root. "${WORK_DIR}/disk${i}"
+
+	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+
+	sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
+    done
+
+    for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
+	"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
+    done
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_4drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/heal-manual.go

@@ -0,0 +1,87 @@
+//go:build ignore
+// +build ignore
+
+//
+// MinIO Object Storage (c) 2022 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New(os.Args[1], os.Args[2], os.Args[3], false)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	opts := madmin.HealOpts{
+		Recursive: true,                  // recursively heal all objects at 'prefix'
+		Remove:    true,                  // remove content that has lost quorum and not recoverable
+		Recreate:  true,                  // rewrite all old non-inlined xl.meta to new xl.meta
+		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
+	}
+
+	start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	fmt.Println("Healstart sequence ===")
+	enc := json.NewEncoder(os.Stdout)
+	if err = enc.Encode(&start); err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Println()
+	for {
+		_, status, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, start.ClientToken, false, false)
+		if status.Summary == "finished" {
+			fmt.Println("Healstatus on items ===")
+			for _, item := range status.Items {
+				if err = enc.Encode(&item); err != nil {
+					log.Fatalln(err)
+				}
+			}
+			break
+		}
+		if status.Summary == "stopped" {
+			fmt.Println("Healstatus on items ===")
+			fmt.Println("Heal failed with", status.FailureDetail)
+			break
+		}
+
+		for _, item := range status.Items {
+			if err = enc.Encode(&item); err != nil {
+				log.Fatalln(err)
+			}
+		}
+
+		time.Sleep(time.Second)
+	}
+}

buildscripts/minio-upgrade.sh

@@ -66,7 +66,7 @@ __init__() {
     mc mb minio/minio-test/
     mc cp ./minio minio/minio-test/to-read/
     mc cp /etc/hosts minio/minio-test/to-read/hosts
-    mc policy set download minio/minio-test
+    mc anonymous set download minio/minio-test
 
     verify_checksum_mc ./minio minio/minio-test/to-read/minio
 

buildscripts/race.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+export GORACE="history_size=7"
+export MINIO_API_REQUESTS_MAX=10000
+
 ## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
 for d in $(go list ./... | grep -v dsync); do
     CGO_ENABLED=1 go test -v -race --timeout 100m "$d"

buildscripts/resolve-right-versions.sh

@@ -39,7 +39,7 @@ function start_minio_5drive() {
     "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
     pid=$!
     disown $pid
-    sleep 30
+    sleep 5
 
     if ! ps -p ${pid} 1>&2 >/dev/null; then
 	echo "server1 log:"

buildscripts/rewrite-old-new.sh

@@ -0,0 +1,151 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
+	chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
+    fi
+}
+
+function verify_rewrite() {
+    start_port=$1
+
+    export MINIO_ACCESS_KEY=minio
+    export MINIO_SECRET_KEY=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export MINIO_CI_CD=1
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 10
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    kill ${pid}
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 10
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -versions \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
+    sleep 1
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -versions \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    kill ${pid}
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    verify_rewrite ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/unaligned-healing.sh

@@ -120,7 +120,7 @@ function start_minio_16drive() {
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
 
 	"${WORK_DIR}/mc" mb play/inspects
 	"${WORK_DIR}/mc" mirror inspects play/inspects
@@ -140,7 +140,7 @@ function start_minio_16drive() {
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
 
 	"${WORK_DIR}/mc" mb play/inspects
 	"${WORK_DIR}/mc" mirror inspects play/inspects

buildscripts/verify-healing-with-root-disks.sh

@@ -0,0 +1,97 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$(mktemp -d)"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+
+function start_minio() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    unset MINIO_CI_CD
+    unset CI
+
+    args=()
+    for i in $(seq 1 4); do
+            args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ")
+    done
+
+    for i in $(seq 1 4); do
+            "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
+    done
+
+    # Wait until all nodes return 403
+    for i in $(seq 1 4); do
+	    while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do
+		    echo -n ".";
+                    sleep 1;
+            done
+    done
+
+ }
+
+# Prepare fake disks with losetup
+function prepare_block_devices() {
+        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+        for i in 1 2 3 4; do
+                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
+                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
+                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
+                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
+                mkdir -p ${WORK_DIR}/mnt/disk$i/
+                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+        done
+}
+
+# Start a distributed MinIO setup, unmount one disk and check if it is formatted
+function main() {
+    start_port=$(shuf -i 10000-65000 -n 1)
+    start_minio ${start_port}
+
+    # Unmount the disk, after the unmount the device id
+    # /tmp/xxx/mnt/disk4 will be the same as '/' and it
+    # will be detected as root disk
+    while [ "$u" != "0" ]; do
+	    sudo umount ${WORK_DIR}/mnt/disk4/
+	    u=$?
+	    sleep 1
+    done
+
+    # Wait until MinIO self heal kicks in
+    sleep 60
+
+    if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
+	    echo "A root disk is formatted unexpectedely"
+            cat "${WORK_DIR}/server4.log"
+            exit -1
+    fi
+}
+
+function cleanup() {
+    pkill minio
+    sudo umount ${WORK_DIR}/mnt/disk{1..3}/
+    sudo rm /dev/minio-loopdisk*
+    rm -rf "$WORK_DIR"
+}
+
+( prepare_block_devices )
+( main "$@" )
+rv=$?
+
+cleanup
+exit "$rv"
+

cmd/acl-handlers.go

@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/admin-bucket-handlers.go

@@ -18,15 +18,30 @@
 package cmd
 
 import (
+	"bytes"
+	"encoding/base64"
 	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/klauspost/compress/zip"
+	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
@@ -47,19 +62,18 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	data, err := ioutil.ReadAll(r.Body)
+	data, err := io.ReadAll(r.Body)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
@@ -76,15 +90,17 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	bucketMeta := madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeQuotaConfig,
-		Bucket: bucket,
-		Quota:  data,
+		Type:      madmin.SRBucketMetaTypeQuotaConfig,
+		Bucket:    bucket,
+		Quota:     data,
+		UpdatedAt: updatedAt,
 	}
 	if quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
@@ -108,14 +124,13 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -145,7 +160,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	bucket := pathClean(vars["bucket"])
 	update := r.Form.Get("update") == "true"
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -153,12 +168,11 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -234,7 +248,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
 		switch err.(type) {
-		case BucketRemoteConnectionErr:
+		case RemoteTargetConnectionErr:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -251,7 +265,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -274,19 +288,19 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 	arnType := vars["type"]
-	if !globalIsErasure {
+
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -314,19 +328,18 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	bucket := pathClean(vars["bucket"])
 	arn := vars["arn"]
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -345,7 +358,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -353,3 +366,794 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	// Write success response.
 	writeSuccessNoContent(w)
 }
+
+// ExportBucketMetadataHandler - exports all bucket metadata as a zipped file
+func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ExportBucketMetadata")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	bucket := pathClean(r.Form.Get("bucket"))
+	if globalIsGateway {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var (
+		buckets []BucketInfo
+		err     error
+	)
+	if bucket != "" {
+		// Check if bucket exists.
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		buckets = append(buckets, BucketInfo{Name: bucket})
+	} else {
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	// Initialize a zip writer which will provide a zipped content
+	// of bucket metadata
+	zipWriter := zip.NewWriter(w)
+	defer zipWriter.Close()
+	rawDataFn := func(r io.Reader, filename string, sz int) error {
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    filename,
+			size:    int64(sz),
+			mode:    0o600,
+			modTime: time.Now(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		header.Method = zip.Deflate
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		if _, err := io.Copy(zwriter, r); err != nil {
+			logger.LogIf(ctx, err)
+		}
+		return nil
+	}
+
+	cfgFiles := []string{
+		bucketPolicyConfig,
+		bucketNotificationConfig,
+		bucketLifecycleConfig,
+		bucketSSEConfig,
+		bucketTaggingConfig,
+		bucketQuotaConfigFile,
+		objectLockConfig,
+		bucketVersioningConfig,
+		bucketReplicationConfig,
+		bucketTargetsFile,
+	}
+	for _, bi := range buckets {
+		for _, cfgFile := range cfgFiles {
+			cfgPath := pathJoin(bi.Name, cfgFile)
+			bucket := bi.Name
+			switch cfgFile {
+			case bucketNotificationConfig:
+				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketLifecycleConfig:
+				config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+						continue
+					}
+					logger.LogIf(ctx, err)
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketQuotaConfigFile:
+				config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
+				if err != nil {
+					if errors.Is(err, BucketQuotaConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				configData, err := json.Marshal(config)
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketSSEConfig:
+				config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketSSEConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketTaggingConfig:
+				config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketTaggingNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case objectLockConfig:
+				config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketVersioningConfig:
+				config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				// ignore empty versioning configs
+				if config.Status != versioning.Enabled && config.Status != versioning.Suspended {
+					continue
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketReplicationConfig:
+				config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+				if err != nil {
+					if errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketTargetsFile:
+				config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketRemoteTargetNotFound{Bucket: bucket}) {
+						continue
+					}
+
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			}
+		}
+	}
+}
+
+type importMetaReport struct {
+	madmin.BucketMetaImportErrs
+}
+
+func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
+	st := i.Buckets[bucket]
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	switch fname {
+	case bucketPolicyConfig:
+		st.Policy = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketNotificationConfig:
+		st.Notification = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketLifecycleConfig:
+		st.Lifecycle = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketSSEConfig:
+		st.SSEConfig = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketTaggingConfig:
+		st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketQuotaConfigFile:
+		st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case objectLockConfig:
+		st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketVersioningConfig:
+		st.Versioning = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	default:
+		st.Err = errMsg
+	}
+	i.Buckets[bucket] = st
+}
+
+// ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config
+// There are some caveats regarding the following:
+// 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here.
+// 2. Replication config - is omitted from import as remote target credentials are not available from exported data for security reasons.
+// 3. lifecycle config - if transition rules are present, tier name needs to have been defined.
+func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ImportBucketMetadata")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	if globalIsGateway {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
+	if objectAPI == nil {
+		return
+	}
+	data, err := io.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	reader := bytes.NewReader(data)
+	zr, err := zip.NewReader(reader, int64(len(data)))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	bucketMap := make(map[string]struct{}, 1)
+	rpt := importMetaReport{
+		madmin.BucketMetaImportErrs{
+			Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
+		},
+	}
+	// import object lock config if any - order of import matters here.
+	for _, file := range zr.File {
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
+		}
+		bucket, fileName := slc[0], slc[1]
+		switch fileName {
+		case objectLockConfig:
+			reader, err := file.Open()
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			config, err := objectlock.ParseObjectLockConfig(reader)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
+			}
+
+			configData, err := xml.Marshal(config)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			if _, ok := bucketMap[bucket]; !ok {
+				opts := MakeBucketOptions{
+					LockEnabled: config.ObjectLockEnabled == "Enabled",
+				}
+				err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
+				if err != nil {
+					if _, ok := err.(BucketExists); !ok {
+						rpt.SetStatus(bucket, fileName, err)
+						continue
+					}
+				}
+				bucketMap[bucket] = struct{}{}
+			}
+
+			// Deny object locking configuration settings on existing buckets without object lock enabled.
+			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+
+			// Call site replication hook.
+			//
+			// We encode the xml bytes as base64 to ensure there are no encoding
+			// errors.
+			cfgStr := base64.StdEncoding.EncodeToString(configData)
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:             madmin.SRBucketMetaTypeObjectLockConfig,
+				Bucket:           bucket,
+				ObjectLockConfig: &cfgStr,
+				UpdatedAt:        updatedAt,
+			}); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+		}
+	}
+
+	// import versioning metadata
+	for _, file := range zr.File {
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
+		}
+		bucket, fileName := slc[0], slc[1]
+		switch fileName {
+		case bucketVersioningConfig:
+			reader, err := file.Open()
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize))
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			if _, ok := bucketMap[bucket]; !ok {
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
+					if _, ok := err.(BucketExists); !ok {
+						rpt.SetStatus(bucket, fileName, err)
+						continue
+					}
+				}
+				bucketMap[bucket] = struct{}{}
+			}
+
+			if globalSiteReplicationSys.isEnabled() && v.Suspended() {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
+				continue
+			}
+
+			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
+				continue
+			}
+			if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
+				continue
+			}
+
+			configData, err := xml.Marshal(v)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
+			}
+
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+		}
+	}
+
+	for _, file := range zr.File {
+		reader, err := file.Open()
+		if err != nil {
+			rpt.SetStatus(file.Name, "", err)
+			continue
+		}
+		sz := file.FileInfo().Size()
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
+		}
+		bucket, fileName := slc[0], slc[1]
+		// create bucket if it does not exist yet.
+		if _, ok := bucketMap[bucket]; !ok {
+			err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+			if err != nil {
+				if _, ok := err.(BucketExists); !ok {
+					rpt.SetStatus(bucket, "", err)
+					continue
+				}
+			}
+			bucketMap[bucket] = struct{}{}
+		}
+		if _, ok := bucketMap[bucket]; !ok {
+			continue
+		}
+		switch fileName {
+		case bucketNotificationConfig:
+			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalEventNotifier.targetList)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
+			}
+
+			configData, err := xml.Marshal(config)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rulesMap := config.ToRulesMap()
+			globalEventNotifier.AddRulesMap(bucket, rulesMap)
+			rpt.SetStatus(bucket, fileName, nil)
+		case bucketPolicyConfig:
+			// Error out if Content-Length is beyond allowed size.
+			if sz > maxBucketPolicySize {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
+				continue
+			}
+
+			bucketPolicyBytes, err := io.ReadAll(io.LimitReader(reader, sz))
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			// Version in policy must not be empty
+			if bucketPolicy.Version == "" {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrMalformedPolicy.String()))
+				continue
+			}
+
+			configData, err := json.Marshal(bucketPolicy)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+			// Call site replication hook.
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:      madmin.SRBucketMetaTypePolicy,
+				Bucket:    bucket,
+				Policy:    bucketPolicyBytes,
+				UpdatedAt: updatedAt,
+			}); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+		case bucketLifecycleConfig:
+			bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			// Validate the received bucket policy document
+			if err = bucketLifecycle.Validate(); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			// Validate the transition storage ARNs
+			if err = validateTransitionTier(bucketLifecycle); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			configData, err := xml.Marshal(bucketLifecycle)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+		case bucketSSEConfig:
+			// Parse bucket encryption xml
+			encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize))
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
+			}
+
+			// Return error if KMS is not initialized
+			if GlobalKMS == nil {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s", errorCodes[ErrKMSNotConfigured].Description))
+				continue
+			}
+			kmsKey := encConfig.KeyID()
+			if kmsKey != "" {
+				kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
+				_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
+				if err != nil {
+					if errors.Is(err, kes.ErrKeyNotFound) {
+						rpt.SetStatus(bucket, fileName, errKMSKeyNotFound)
+						continue
+					}
+					rpt.SetStatus(bucket, fileName, err)
+					continue
+				}
+			}
+
+			configData, err := xml.Marshal(encConfig)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			// Store the bucket encryption configuration in the object layer
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+
+			// Call site replication hook.
+			//
+			// We encode the xml bytes as base64 to ensure there are no encoding
+			// errors.
+			cfgStr := base64.StdEncoding.EncodeToString(configData)
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:      madmin.SRBucketMetaTypeSSEConfig,
+				Bucket:    bucket,
+				SSEConfig: &cfgStr,
+				UpdatedAt: updatedAt,
+			}); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+		case bucketTaggingConfig:
+			tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
+			}
+
+			configData, err := xml.Marshal(tags)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+
+			// Call site replication hook.
+			//
+			// We encode the xml bytes as base64 to ensure there are no encoding
+			// errors.
+			cfgStr := base64.StdEncoding.EncodeToString(configData)
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:      madmin.SRBucketMetaTypeTags,
+				Bucket:    bucket,
+				Tags:      &cfgStr,
+				UpdatedAt: updatedAt,
+			}); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+		case bucketQuotaConfigFile:
+			data, err := io.ReadAll(reader)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			quotaConfig, err := parseBucketQuota(bucket, data)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+
+			if quotaConfig.Type == "fifo" {
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
+				continue
+			}
+
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
+			if err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+			rpt.SetStatus(bucket, fileName, nil)
+
+			bucketMeta := madmin.SRBucketMeta{
+				Type:      madmin.SRBucketMetaTypeQuotaConfig,
+				Bucket:    bucket,
+				Quota:     data,
+				UpdatedAt: updatedAt,
+			}
+			if quotaConfig.Quota == 0 {
+				bucketMeta.Quota = nil
+			}
+
+			// Call site replication hook.
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
+				rpt.SetStatus(bucket, fileName, err)
+				continue
+			}
+		}
+	}
+
+	rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, rptData)
+}
+
+// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
+// to the connected HTTP client. This is a MinIO only extension
+func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ReplicationDiff")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if globalIsGateway {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	opts := extractReplicateDiffOpts(r.Form)
+	if opts.ARN != "" {
+		tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, opts.ARN)
+		if tgt.Empty() {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, fmt.Errorf("invalid arn : '%s'", opts.ARN)), r.URL)
+			return
+		}
+	}
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	diffCh, err := getReplicationDiff(ctx, objectAPI, bucket, opts)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	enc := json.NewEncoder(w)
+	for {
+		select {
+		case entry, ok := <-diffCh:
+			if !ok {
+				return
+			}
+			if err := enc.Encode(entry); err != nil {
+				return
+			}
+			if len(diffCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
+		case <-keepAliveTicker.C:
+			if len(diffCh) > 0 {
+				continue
+			}
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
+	}
+}

cmd/admin-handler-utils.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 
 	"github.com/minio/kes"
@@ -29,6 +30,10 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
+// validateAdminReq will validate request against and return whether it is allowed.
+// If any of the supplied actions are allowed it will be successful.
+// If nil ObjectLayer is returned, the operation is not permitted.
+// When nil ObjectLayer has been returned an error has always been sent to w.
 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -40,11 +45,16 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	for _, action := range actions {
 		// Validate request signature.
 		cred, adminAPIErr := checkAdminRequestAuth(ctx, r, action, "")
-		if adminAPIErr != ErrNone {
+		switch adminAPIErr {
+		case ErrNone:
+			return objectAPI, cred
+		case ErrAccessDenied:
+			// Try another
+			continue
+		default:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 			return nil, cred
 		}
-		return objectAPI, cred
 	}
 	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 	return nil, auth.Credentials{}
@@ -96,6 +106,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 		}
 	default:
 		switch {
+		case errors.Is(err, errTooManyPolicies):
+			apiErr = APIError{
+				Code:           "XMinioAdminInvalidRequest",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errDecommissionAlreadyRunning):
 			apiErr = APIError{
 				Code:           "XMinioDecommissionNotAllowed",
@@ -180,7 +196,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			apiErr = APIError{
 				Code:           "XMinioAdminTierBackendInUse",
 				Description:    err.Error(),
-				HTTPStatusCode: http.StatusConflict,
+				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case errors.Is(err, errTierBackendNotEmpty):
 			apiErr = APIError{
@@ -217,3 +233,27 @@ func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
 		return toAPIErrorCode(ctx, err)
 	}
 }
+
+// wraps export error for more context
+func exportError(ctx context.Context, err error, fname, entity string) APIError {
+	if entity == "" {
+		return toAPIError(ctx, fmt.Errorf("error exporting %s with: %w", fname, err))
+	}
+	return toAPIError(ctx, fmt.Errorf("error exporting %s from %s with: %w", entity, fname, err))
+}
+
+// wraps import error for more context
+func importError(ctx context.Context, err error, fname, entity string) APIError {
+	if entity == "" {
+		return toAPIError(ctx, fmt.Errorf("error importing %s with: %w", fname, err))
+	}
+	return toAPIError(ctx, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err))
+}
+
+// wraps import error for more context
+func importErrorWithAPIErr(ctx context.Context, apiErr APIErrorCode, err error, fname, entity string) APIError {
+	if entity == "" {
+		return errorCodes.ToAPIErrWithErr(apiErr, fmt.Errorf("error importing %s with: %w", fname, err))
+	}
+	return errorCodes.ToAPIErrWithErr(apiErr, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err))
+}

cmd/admin-handlers-config-kv.go

@@ -33,7 +33,8 @@ import (
 	"github.com/minio/minio/internal/config/etcd"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
-	"github.com/minio/minio/internal/config/policy/opa"
+	idplugin "github.com/minio/minio/internal/config/identity/plugin"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -80,6 +81,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
 	if err = validateConfig(cfg, subSys); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
@@ -90,6 +92,12 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// freshly retrieve the config so that default values are loaded for reset config
+	if cfg, err = getValidConfig(objectAPI); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	dynamic := config.SubSystemsDynamic.Contains(subSys)
 	if dynamic {
 		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
@@ -104,7 +112,7 @@ func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config,
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	globalNotificationSys.SignalService(serviceReloadDynamic)
+	globalNotificationSys.SignalConfigReload(subSys)
 	// Tell the client that dynamic config was applied.
 	w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 }
@@ -172,10 +180,16 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	if dynamic {
 		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
 	}
+
 	writeSuccessResponseHeadersOnly(w)
 }
 
 // GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
+//
+// `key` can be one of three forms:
+// 1. `subsys:target` -> request for config of a single subsystem and target pair.
+// 2. `subsys:` -> request for config of a single subsystem and the default target.
+// 3. `subsys` -> request for config of all targets for the given subsystem.
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
@@ -188,15 +202,34 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	cfg := globalServerConfig.Clone()
 	vars := mux.Vars(r)
-	buf := &bytes.Buffer{}
-	cw := config.NewConfigWriteTo(cfg, vars["key"])
-	if _, err := cw.WriteTo(buf); err != nil {
+	key := vars["key"]
+
+	var subSys, target string
+	{
+		ws := strings.SplitN(key, madmin.SubSystemSeparator, 2)
+		subSys = ws[0]
+		if len(ws) == 2 {
+			if ws[1] == "" {
+				target = madmin.Default
+			} else {
+				target = ws[1]
+			}
+		}
+	}
+
+	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	var s strings.Builder
+	for _, subSysConfig := range subSysConfigs {
+		subSysConfig.AddString(&s, false)
+	}
+
 	password := cred.SecretKey
-	econfigData, err := madmin.EncryptData(password, buf.Bytes())
+	econfigData, err := madmin.EncryptData(password, []byte(s.String()))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -426,40 +459,30 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		count += len(cfg[hkv.Key])
 	}
 	for _, hkv := range hkvs {
-		v := cfg[hkv.Key]
-		for target, kv := range v {
-			off := kv.Get(config.Enable) == config.EnableOff
+		// We ignore the error below, as we cannot get one.
+		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
+
+		for _, item := range cfgSubsysItems {
+			off := item.Config.Get(config.Enable) == config.EnableOff
 			switch hkv.Key {
 			case config.EtcdSubSys:
-				off = !etcd.Enabled(kv)
+				off = !etcd.Enabled(item.Config)
 			case config.CacheSubSys:
-				off = !cache.Enabled(kv)
+				off = !cache.Enabled(item.Config)
 			case config.StorageClassSubSys:
-				off = !storageclass.Enabled(kv)
-			case config.PolicyOPASubSys:
-				off = !opa.Enabled(kv)
+				off = !storageclass.Enabled(item.Config)
+			case config.PolicyPluginSubSys:
+				off = !polplugin.Enabled(item.Config)
 			case config.IdentityOpenIDSubSys:
-				off = !openid.Enabled(kv)
+				off = !openid.Enabled(item.Config)
 			case config.IdentityLDAPSubSys:
-				off = !xldap.Enabled(kv)
+				off = !xldap.Enabled(item.Config)
 			case config.IdentityTLSSubSys:
 				off = !globalSTSTLSConfig.Enabled
+			case config.IdentityPluginSubSys:
+				off = !idplugin.Enabled(item.Config)
 			}
-			if off {
-				s.WriteString(config.KvComment)
-				s.WriteString(config.KvSpaceSeparator)
-			}
-			s.WriteString(hkv.Key)
-			if target != config.Default {
-				s.WriteString(config.SubSystemSeparator)
-				s.WriteString(target)
-			}
-			s.WriteString(config.KvSpaceSeparator)
-			s.WriteString(kv.String())
-			count--
-			if count > 0 {
-				s.WriteString(config.KvNewline)
-			}
+			item.AddString(&s, off)
 		}
 	}
 

cmd/admin-handlers-idp-config.go

@@ -0,0 +1,321 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// List of implemented ID config types.
+var idCfgTypes = set.CreateStringSet("openid")
+
+// SetIdentityProviderCfg:
+//
+// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
+func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetIdentityCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	var cfgDataBuilder strings.Builder
+	switch cfgType {
+	case "openid":
+		fmt.Fprintf(&cfgDataBuilder, "identity_openid")
+	}
+
+	// Ensure body content type is opaque.
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/octet-stream" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+		return
+	}
+
+	// Subsystem configuration name could be empty.
+	cfgName := mux.Vars(r)["name"]
+	if cfgName != "" {
+		fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
+	}
+
+	fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
+
+	cfgData := cfgDataBuilder.String()
+	subSys, _, _, err := config.GetSubSys(cfgData)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	dynamic, err := cfg.ReadConfig(strings.NewReader(cfgData))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// IDP config is not dynamic. Sanity check.
+	if dynamic {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+
+	// Update the actual server config on disk.
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write to the config input KV to history.
+	if err = saveServerConfigHistory(ctx, objectAPI, []byte(cfgData)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetIdentityProviderCfg:
+//
+// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
+//
+// GetIdentityProviderCfg returns a list of configured IDPs on the server if
+// name is empty. If name is non-empty, returns the configuration details for
+// the IDP of the given type and configuration name. The configuration name for
+// the default ("un-named") configuration target is `_`.
+func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetIdentityProviderCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	cfgName := r.Form.Get("name")
+	password := cred.SecretKey
+
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// If no cfgName is provided, we list.
+	if cfgName == "" {
+		a.listIdentityProviders(ctx, w, r, cfgType, password)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+
+	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	if err != nil {
+		if err == openid.ErrProviderConfigNotFound {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	res := madmin.IDPConfig{
+		Type: cfgType,
+		Name: cfgName,
+		Info: cfgInfos,
+	}
+	data, err := json.Marshal(res)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
+	// var subSys string
+	switch cfgType {
+	case "openid":
+		// subSys = config.IdentityOpenIDSubSys
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+	cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(cfgList)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// DeleteIdentityProviderCfg:
+//
+// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
+func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteIdentityProviderCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	cfgName := mux.Vars(r)["name"]
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+
+	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	if err != nil {
+		if err == openid.ErrProviderConfigNotFound {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	hasEnv := false
+	for _, ci := range cfgInfos {
+		if ci.IsCfg && ci.IsEnv {
+			hasEnv = true
+			break
+		}
+	}
+
+	if hasEnv {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+		return
+	}
+
+	var subSys string
+	switch cfgType {
+	case "openid":
+		subSys = config.IdentityOpenIDSubSys
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg, err = readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	dynamic := config.SubSystemsDynamic.Contains(subSys)
+	if dynamic {
+		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+	}
+}

cmd/admin-handlers-pools.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -149,8 +150,10 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
+		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
+		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
 		// We didn't find any matching pools, invalid input
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
 		return
 	}
 

cmd/admin-handlers-site-replication.go

@@ -22,8 +22,9 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -116,19 +117,38 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
 		_, isForceCreate := r.Form["forceCreate"]
-		opts := BucketOptions{
+		createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
+		createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
+		if cerr != nil {
+			createdAt = timeSentinel
+		}
+
+		opts := MakeBucketOptions{
 			Location:          r.Form.Get("location"),
 			LockEnabled:       isLockEnabled,
 			VersioningEnabled: isVersioningEnabled,
 			ForceCreate:       isForceCreate,
+			CreatedAt:         createdAt,
 		}
 		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 	case madmin.ConfigureReplBktOp:
 		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
 	case madmin.DeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      false,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
 	case madmin.ForceDeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      true,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
+	case madmin.PurgeDeletedBucketOp:
+		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -160,7 +180,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRIAMItemPolicy:
 		if item.Policy == nil {
-			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 		} else {
 			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
 			if perr != nil {
@@ -168,21 +188,21 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 				return
 			}
 			if policy.IsEmpty() {
-				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 			} else {
-				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy, item.UpdatedAt)
 			}
 		}
 	case madmin.SRIAMItemSvcAcc:
-		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange)
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
 	case madmin.SRIAMItemPolicyMapping:
-		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
 	case madmin.SRIAMItemSTSAcc:
-		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
 	case madmin.SRIAMItemIAMUser:
-		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser)
+		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
 	case madmin.SRIAMItemGroupInfo:
-		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo)
+		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -214,7 +234,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRBucketMetaTypePolicy:
 		if item.Policy == nil {
-			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
 			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
 			if berr != nil {
@@ -222,31 +242,33 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				return
 			}
 			if bktPolicy.IsEmpty() {
-				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 			} else {
-				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy, item.UpdatedAt)
 			}
 		}
 	case madmin.SRBucketMetaTypeQuotaConfig:
 		if item.Quota == nil {
-			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil)
+			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
 			quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
-			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig); err != nil {
+			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 		}
+	case madmin.SRBucketMetaTypeVersionConfig:
+		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeTags:
-		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
+		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
-		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
+		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeSSEConfig:
-		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
+		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -296,7 +318,7 @@ func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Re
 }
 
 func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
-	data, err := ioutil.ReadAll(body)
+	data, err := io.ReadAll(body)
 	if err != nil {
 		return SRError{
 			Cause: err,
@@ -434,6 +456,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	opts.Users = q.Get("users") == "true"
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
+	opts.ShowDeleted = q.Get("showDeleted") == "true"
 	return
 }
 

cmd/admin-handlers-users-race_test.go

@@ -26,6 +26,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -41,11 +42,15 @@ func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
 }
 
 func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip("windows is clunky")
+	}
+
 	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on ErasureSD backend with signature v4.
+		{serverType: "ErasureSD", signer: signerV4},
+		// Init and run test on ErasureSD backend, with tls enabled.
+		{serverType: "ErasureSD", signer: signerV4, secure: true},
 		// Init and run test on Erasure backend.
 		{serverType: "Erasure", signer: signerV4},
 		// Init and run test on ErasureSet backend.

cmd/admin-handlers-users_test.go

@@ -24,16 +24,17 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/minio/madmin-go"
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -97,6 +98,29 @@ func (s *TestSuiteIAM) iamSetup(c *check) {
 	}
 }
 
+// List of all IAM test suites (i.e. test server configuration combinations)
+// common to tests.
+var iamTestSuites = func() []*TestSuiteIAM {
+	baseTestCases := []TestSuiteCommon{
+		// Init and run test on ErasureSD backend with signature v4.
+		{serverType: "ErasureSD", signer: signerV4},
+		// Init and run test on ErasureSD backend, with tls enabled.
+		{serverType: "ErasureSD", signer: signerV4, secure: true},
+		// Init and run test on Erasure backend.
+		{serverType: "Erasure", signer: signerV4},
+		// Init and run test on ErasureSet backend.
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
+	}
+	return testCases
+}()
+
 const (
 	EnvTestEtcdBackend = "ETCD_SERVER"
 )
@@ -166,30 +190,12 @@ func (s *TestSuiteIAM) getUserClient(c *check, accessKey, secretKey, sessionToke
 }
 
 func TestIAMInternalIDPServerSuite(t *testing.T) {
-	baseTestCases := []TestSuiteCommon{
-		// Init and run test on FS backend with signature v4.
-		{serverType: "FS", signer: signerV4},
-		// Init and run test on FS backend, with tls enabled.
-		{serverType: "FS", signer: signerV4, secure: true},
-		// Init and run test on Erasure backend.
-		{serverType: "Erasure", signer: signerV4},
-		// Init and run test on ErasureSet backend.
-		{serverType: "ErasureSet", signer: signerV4},
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip("windows is clunky disable these tests")
 	}
-	testCases := []*TestSuiteIAM{}
-	for _, bt := range baseTestCases {
-		testCases = append(testCases,
-			newTestSuiteIAM(bt, false),
-			newTestSuiteIAM(bt, true),
-		)
-	}
-	for i, testCase := range testCases {
-		etcdStr := ""
-		if testCase.withEtcdBackend {
-			etcdStr = " (with etcd backend)"
-		}
+	for i, testCase := range iamTestSuites {
 		t.Run(
-			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
 			func(t *testing.T) {
 				suite := testCase
 				c := &check{t, testCase.serverType}
@@ -236,6 +242,7 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	if err != nil {
 		c.Fatalf("unable to set policy: %v", err)
 	}
+
 	client := s.getUserClient(c, accessKey, secretKey, "")
 	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
 	if err != nil {
@@ -381,7 +388,7 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
 	req.ContentLength = int64(len(buf))
 	sum := sha256.Sum256(buf)
 	req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
-	req.Body = ioutil.NopCloser(bytes.NewReader(buf))
+	req.Body = io.NopCloser(bytes.NewReader(buf))
 	req = signer.SignV4(*req, accessKey, secretKey, "", "")
 
 	// 3.1 Execute the request.
@@ -973,7 +980,168 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	pluginEndpoint := os.Getenv("POLICY_PLUGIN_ENDPOINT")
+	if pluginEndpoint == "" {
+		c.Skip("POLICY_PLUGIN_ENDPOINT not given - skipping.")
+	}
+
+	configCmds := []string{
+		"policy_plugin",
+		"url=" + pluginEndpoint,
+	}
+
+	_, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " "))
+	if err != nil {
+		c.Fatalf("unable to setup access management plugin for tests: %v", err)
+	}
+
+	s.RestartIAMSuite(c)
+}
+
+// TestIAM_AMPInternalIDPServerSuite - tests for access management plugin
+func TestIAM_AMPInternalIDPServerSuite(t *testing.T) {
+	for i, testCase := range iamTestSuites {
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription),
+			func(t *testing.T) {
+				suite := testCase
+				c := &check{t, testCase.serverType}
+
+				suite.SetUpSuite(c)
+				defer suite.TearDownSuite(c)
+
+				suite.SetUpAccMgmtPlugin(c)
+
+				suite.TestAccMgmtPlugin(c)
+			},
+		)
+	}
+}
+
+// TestAccMgmtPlugin - this test assumes that the access-management-plugin is
+// the same as the example in `docs/iam/access-manager-plugin.go` -
+// specifically, it denies only `s3:Put*` operations on non-root accounts.
+func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	// 0. Check that owner is able to make-bucket.
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// 1. Create a user.
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	// 2. Check new user appears in listing
+	usersMap, err := s.adm.ListUsers(ctx)
+	if err != nil {
+		c.Fatalf("error listing: %v", err)
+	}
+	v, ok := usersMap[accessKey]
+	if !ok {
+		c.Fatalf("user not listed: %s", accessKey)
+	}
+	c.Assert(v.Status, madmin.AccountEnabled)
+
+	// 3. Check that user is able to make a bucket.
+	client := s.getUserClient(c, accessKey, secretKey, "")
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("user not create bucket: %v", err)
+	}
+
+	// 3.1 check user has access to bucket
+	c.mustListObjects(ctx, client, bucket)
+
+	// 3.2 check that user cannot upload an object.
+	_, err = client.PutObject(ctx, bucket, "objectName", bytes.NewBuffer([]byte("some content")), 12, minio.PutObjectOptions{})
+	if err == nil {
+		c.Fatalf("user was able to upload unexpectedly")
+	}
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  cr.NewStaticV4(accessKey, secretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Create svc acc
+	cr := c.mustCreateSvcAccount(ctx, accessKey, userAdmClient)
+
+	// 1. Check that svc account appears in listing
+	c.assertSvcAccAppearsInListing(ctx, userAdmClient, accessKey, cr.AccessKey)
+
+	// 2. Check that svc account info can be queried
+	c.assertSvcAccInfoQueryable(ctx, userAdmClient, accessKey, cr.AccessKey, false)
+
+	// 3. Check S3 access
+	c.assertSvcAccS3Access(ctx, s, cr, bucket)
+
+	// Check that session policies do not apply - as policy enforcement is
+	// delegated to plugin.
+	{
+		svcAK, svcSK := mustGenerateCredentials(c)
+
+		// This policy does not allow listing objects.
+		policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+		cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+			Policy:     policyBytes,
+			TargetUser: accessKey,
+			AccessKey:  svcAK,
+			SecretKey:  svcSK,
+		})
+		if err != nil {
+			c.Fatalf("Unable to create svc acc: %v", err)
+		}
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		// Though the attached policy does not allow listing, it will be
+		// ignored because the plugin allows it.
+		c.mustListObjects(ctx, svcClient, bucket)
+	}
+
+	// 4. Check that service account's secret key and account status can be
+	// updated.
+	c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket)
+
+	// 5. Check that service account can be deleted.
+	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
+
+	// 6. Check that service account **can** be created for some other user.
+	// This is possible because of the policy enforced in the plugin.
+	c.mustCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
+}
+
 func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
+	c.Helper()
 	randUser := mustGetUUID()
 	randPass := mustGetUUID()
 	err := admClnt.AddUser(ctx, randUser, randPass)
@@ -987,6 +1155,7 @@ func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClie
 }
 
 func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo {
+	c.Helper()
 	ui, err := admClnt.GetUserInfo(ctx, accessKey)
 	if err != nil {
 		c.Fatalf("should be able to get user info: %v", err)
@@ -995,6 +1164,7 @@ func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminCli
 }
 
 func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) {
+	c.Helper()
 	randUser := mustGetUUID()
 	randPass := mustGetUUID()
 	err := admClnt.AddUser(ctx, randUser, randPass)
@@ -1004,6 +1174,7 @@ func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminC
 }
 
 func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials {
+	c.Helper()
 	cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: tgtUser,
 	})
@@ -1014,6 +1185,7 @@ func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admCln
 }
 
 func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) {
+	c.Helper()
 	_, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: tgtUser,
 	})
@@ -1023,28 +1195,77 @@ func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, adm
 }
 
 func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
 	v, ok := <-res
 	if !ok || v.Err == nil {
-		c.Fatalf("user was able to list unexpectedly!")
+		c.Fatalf("user was able to list unexpectedly! on %s", bucket)
+	}
+}
+
+func (c *check) mustPutObjectWithTags(ctx context.Context, client *minio.Client, bucket, object string) {
+	c.Helper()
+	_, err := client.PutObject(ctx, bucket, object, bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{
+		UserTags: map[string]string{
+			"security": "public",
+			"virus":    "true",
+		},
+	})
+	if err != nil {
+		c.Fatalf("user was unable to upload the object: %v", err)
+	}
+}
+
+func (c *check) mustGetObject(ctx context.Context, client *minio.Client, bucket, object string) {
+	c.Helper()
+
+	r, err := client.GetObject(ctx, bucket, object, minio.GetObjectOptions{})
+	if err != nil {
+		c.Fatalf("user was unable to download the object: %v", err)
+	}
+	defer r.Close()
+
+	_, err = io.Copy(io.Discard, r)
+	if err != nil {
+		c.Fatalf("user was unable to download the object: %v", err)
 	}
 }
 
 func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
 	v, ok := <-res
 	if ok && v.Err != nil {
-		msg := fmt.Sprintf("user was unable to list: %v", v.Err)
-		c.Fatalf(msg)
+		c.Fatalf("user was unable to list: %v", v.Err)
 	}
 }
 
+func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
+	c.Helper()
+	_, err := client.ListBuckets(ctx)
+	if err != nil {
+		c.Fatalf("user was unable to list buckets: %v", err)
+	}
+}
+
+func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
+	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
+	if e, ok := err.(minio.ErrorResponse); ok {
+		if e.Code == "AccessDenied" {
+			return
+		}
+	}
+	c.Fatalf("upload did not get an AccessDenied error - got %#v instead", err)
+}
+
 func (c *check) assertSvcAccS3Access(ctx context.Context, s *TestSuiteIAM, cr madmin.Credentials, bucket string) {
 	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
 	c.mustListObjects(ctx, svcClient, bucket)
 }
 
 func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *madmin.AdminClient, parentAK, svcAK string) {
+	c.Helper()
 	listResp, err := madmClient.ListServiceAccounts(ctx, parentAK)
 	if err != nil {
 		c.Fatalf("unable to list svc accounts: %v", err)
@@ -1070,6 +1291,7 @@ func (c *check) assertSvcAccInfoQueryable(ctx context.Context, madmClient *madmi
 // bucket. It creates a session policy that restricts listing on the bucket and
 // then enables it again in a session policy update call.
 func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
+	c.Helper()
 	svcAK, svcSK := mustGenerateCredentials(c)
 
 	// This policy does not allow listing objects.
@@ -1125,6 +1347,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
 }
 
 func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
+	c.Helper()
 	svcAK, svcSK := mustGenerateCredentials(c)
 	cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: accessKey,
@@ -1161,6 +1384,7 @@ func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *Tes
 }
 
 func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
+	c.Helper()
 	svcAK, svcSK := mustGenerateCredentials(c)
 	cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: accessKey,
@@ -1181,6 +1405,7 @@ func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmC
 }
 
 func mustGenerateCredentials(c *check) (string, string) {
+	c.Helper()
 	ak, sk, err := auth.GenerateCredentials()
 	if err != nil {
 		c.Fatalf("unable to generate credentials: %v", err)

cmd/admin-handlers_test.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -220,7 +219,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	adminTestBed.router.ServeHTTP(rec, req)
 
 	if rec.Code != http.StatusOK {
-		resp, _ := ioutil.ReadAll(rec.Body)
+		resp, _ := io.ReadAll(rec.Body)
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}

cmd/admin-heal-ops.go

@@ -91,8 +91,11 @@ type allHealState struct {
 	sync.RWMutex
 
 	// map of heal path to heal sequence
-	healSeqMap     map[string]*healSequence // Indexed by endpoint
-	healLocalDisks map[Endpoint]struct{}
+	healSeqMap map[string]*healSequence // Indexed by endpoint
+	// keep track of the healing status of disks in the memory
+	//   false: the disk needs to be healed but no healing routine is started
+	//    true: the disk is currently healing
+	healLocalDisks map[Endpoint]bool
 	healStatus     map[string]healingTracker // Indexed by disk ID
 }
 
@@ -100,7 +103,7 @@ type allHealState struct {
 func newHealState(cleanup bool) *allHealState {
 	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
-		healLocalDisks: map[Endpoint]struct{}{},
+		healLocalDisks: make(map[Endpoint]bool),
 		healStatus:     make(map[string]healingTracker),
 	}
 	if cleanup {
@@ -109,13 +112,6 @@ func newHealState(cleanup bool) *allHealState {
 	return hstate
 }
 
-func (ahs *allHealState) healDriveCount() int {
-	ahs.RLock()
-	defer ahs.RUnlock()
-
-	return len(ahs.healLocalDisks)
-}
-
 func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
 	ahs.Lock()
 	defer ahs.Unlock()
@@ -165,23 +161,34 @@ func (ahs *allHealState) getLocalHealingDisks() map[string]madmin.HealingDisk {
 	return dst
 }
 
+// getHealLocalDiskEndpoints() returns the list of disks that need
+// to be healed but there is no healing routine in progress on them.
 func (ahs *allHealState) getHealLocalDiskEndpoints() Endpoints {
 	ahs.RLock()
 	defer ahs.RUnlock()
 
 	var endpoints Endpoints
-	for ep := range ahs.healLocalDisks {
-		endpoints = append(endpoints, ep)
+	for ep, healing := range ahs.healLocalDisks {
+		if !healing {
+			endpoints = append(endpoints, ep)
+		}
 	}
 	return endpoints
 }
 
+func (ahs *allHealState) markDiskForHealing(ep Endpoint) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	ahs.healLocalDisks[ep] = true
+}
+
 func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
 	ahs.Lock()
 	defer ahs.Unlock()
 
 	for _, ep := range healLocalDisks {
-		ahs.healLocalDisks[ep] = struct{}{}
+		ahs.healLocalDisks[ep] = false
 	}
 }
 
@@ -194,7 +201,6 @@ func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	for {
 		select {
 		case <-periodicTimer.C:
-			periodicTimer.Reset(time.Minute * 5)
 			now := UTCNow()
 			ahs.Lock()
 			for path, h := range ahs.healSeqMap {
@@ -203,6 +209,8 @@ func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 				}
 			}
 			ahs.Unlock()
+
+			periodicTimer.Reset(time.Minute * 5)
 		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately
@@ -581,12 +589,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	// heal-results in memory and the client has not consumed it
 	// for too long.
 	unconsumedTimer := time.NewTimer(healUnconsumedTimeout)
-	defer func() {
-		// stop the timeout timer so it is garbage collected.
-		if !unconsumedTimer.Stop() {
-			<-unconsumedTimer.C
-		}
-	}()
+	defer unconsumedTimer.Stop()
 
 	var itemsLen int
 	for {
@@ -808,16 +811,6 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 	}
 }
 
-// healDiskFormat - heals format.json, return value indicates if a
-// failure error occurred.
-func (h *healSequence) healDiskFormat() error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
-	return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
-}
-
 // healBuckets - check for all buckets heal or just particular bucket.
 func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	if h.isQuitting() {
@@ -829,7 +822,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	buckets, err := objAPI.ListBuckets(h.ctx)
+	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}

cmd/admin-router.go

@@ -66,6 +66,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
+		// Metrics operation
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(gz(httpTraceAll(adminAPI.MetricsHandler)))
 
 		if globalIsDistErasure || globalIsErasure {
 			// Heal operations
@@ -170,50 +172,72 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// Set Group Status
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetGroupStatus))).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
-		if globalIsDistErasure || globalIsErasure {
-			// GetBucketQuotaConfig
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
-			// PutBucketQuotaConfig
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+		// Export IAM info to zipped file
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(httpTraceHdrs(adminAPI.ExportIAM))
 
-			// Bucket replication operations
-			// GetBucketTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
-			// RemoveRemoteTargetHandler
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-				gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+		// Import IAM info
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
 
-			// Remote Tier management operations
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
-			// Tier stats
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
+		// IDentity Provider configuration APIs
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
 
-			// Cluster Replication APIs
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
+		// -- END IAM APIs --
 
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
-		}
+		// GetBucketQuotaConfig
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+		// PutBucketQuotaConfig
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+
+		// Bucket replication operations
+		// GetBucketTargetHandler
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+		// SetRemoteTargetHandler
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
+		// RemoveRemoteTargetHandler
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+		// ReplicationDiff - MinIO extension API
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
+
+		// Bucket migration operations
+		// ExportBucketMetaHandler
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ExportBucketMetadataHandler)))
+		// ImportBucketMetaHandler
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-bucket-metadata").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ImportBucketMetadataHandler)))
+
+		// Remote Tier management operations
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
+		// Tier stats
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
+
+		// Cluster Replication APIs
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
+
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
 
 		if globalIsDistErasure {
 			// Top locks
@@ -223,8 +247,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
 		}
 
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
 

cmd/admin-server-info.go

@@ -31,7 +31,10 @@ import (
 // local endpoints from given list of endpoints
 func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
 	var localEndpoints Endpoints
-	addr := r.Host
+	addr := globalLocalNodeName
+	if r != nil {
+		addr = r.Host
+	}
 	if globalIsDistErasure {
 		addr = globalLocalNodeName
 	}
@@ -40,7 +43,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		for _, endpoint := range ep.Endpoints {
 			nodeName := endpoint.Host
 			if nodeName == "" {
-				nodeName = r.Host
+				nodeName = addr
 			}
 			if endpoint.IsLocal {
 				// Only proceed for local endpoints

cmd/api-errors.go

@@ -30,7 +30,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"google.golang.org/api/googleapi"
 
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -131,7 +131,7 @@ const (
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrReplicationDenyEditError
-	ErrReplicationNoMatchingRuleError
+	ErrReplicationNoExistingObjects
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -176,6 +176,7 @@ const (
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
 	ErrBucketAlreadyExists
+	ErrTooManyBuckets
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
 	ErrMaximumExpires
@@ -196,8 +197,9 @@ const (
 	ErrInvalidTagDirective
 	// Add new error codes here.
 
-	// SSE-S3 related API errors
+	// SSE-S3/SSE-KMS related API errors
 	ErrInvalidEncryptionMethod
+	ErrInvalidEncryptionKeyID
 
 	// Server-Side-Encryption (with Customer provided key) related API errors.
 	ErrInsecureSSECustomerRequest
@@ -231,6 +233,7 @@ const (
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
+	ErrContentChecksumMismatch
 
 	// Add new extended error codes here.
 
@@ -267,6 +270,8 @@ const (
 	ErrAdminConfigNoQuorum
 	ErrAdminConfigTooLarge
 	ErrAdminConfigBadJSON
+	ErrAdminNoSuchConfigTarget
+	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
@@ -280,6 +285,7 @@ const (
 	ErrSiteReplicationBucketConfigError
 	ErrSiteReplicationBucketMetaError
 	ErrSiteReplicationIAMError
+	ErrSiteReplicationConfigMissing
 
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
@@ -388,6 +394,8 @@ const (
 	ErrAccountNotEligible
 	ErrAdminServiceAccountNotFound
 	ErrPostPolicyConditionInvalidFormat
+
+	ErrInvalidChecksum
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -677,6 +685,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrTooManyBuckets: {
+		Code:           "TooManyBuckets",
+		Description:    "You have attempted to create more buckets than allowed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
 		Description:    "The bucket you tried to delete is not empty",
@@ -884,7 +897,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
-		Description:    "Remote service connection error - please check remote service credentials and target bucket",
+		Description:    "Remote service connection error",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationBandwidthLimitError: {
@@ -892,15 +905,15 @@ var errorCodes = errorCodeMap{
 		Description:    "Bandwidth limit for remote target must be atleast 100MBps",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationNoMatchingRuleError: {
-		Code:           "XMinioReplicationNoMatchingRule",
-		Description:    "No matching replication rule found for this object prefix",
+	ErrReplicationNoExistingObjects: {
+		Code:           "XMinioReplicationNoExistingObjects",
+		Description:    "No matching ExistingsObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationDenyEditError: {
 		Code:           "XMinioReplicationDenyEdit",
 		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
@@ -1069,6 +1082,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The encryption method specified is not supported",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidEncryptionKeyID: {
+		Code:           "InvalidRequest",
+		Description:    "The specified KMS KeyID contains unsupported characters",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInsecureSSECustomerRequest: {
 		Code:           "InvalidRequest",
 		Description:    "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
@@ -1151,11 +1169,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrContentChecksumMismatch: {
+		Code:           "XAmzContentChecksumMismatch",
+		Description:    "The provided 'x-amz-checksum' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// MinIO extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
-		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
+		Description:    "Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
 	ErrRequestBodyParse: {
@@ -1166,7 +1189,7 @@ var errorCodes = errorCodeMap{
 	ErrObjectExistsAsDirectory: {
 		Code:           "XMinioObjectExistsAsDirectory",
 		Description:    "Object name already exists as a directory.",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidObjectName: {
 		Code:           "XMinioInvalidObjectName",
@@ -1239,11 +1262,21 @@ var errorCodes = errorCodeMap{
 			maxEConfigJSONSize),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoSuchConfigTarget: {
+		Code:           "XMinioAdminNoSuchConfigTarget",
+		Description:    "No such named configuration target exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigBadJSON: {
 		Code:           "XMinioAdminConfigBadJSON",
 		Description:    "JSON configuration provided is of incorrect format",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigEnvOverridden: {
+		Code:           "XMinioAdminConfigEnvOverridden",
+		Description:    "Unable to update config via Admin API due to environment variable override",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigDuplicateKeys: {
 		Code:           "XMinioAdminConfigDuplicateKeys",
 		Description:    "JSON configuration provided has objects with duplicate keys",
@@ -1340,7 +1373,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Error while replicating an IAM item",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-
+	ErrSiteReplicationConfigMissing: {
+		Code:           "XMinioSiteReplicationConfigMissingError",
+		Description:    "Site not found in site replication configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
@@ -1851,6 +1888,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid according to Policy: Policy Condition failed",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidChecksum: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid checksum provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -1904,6 +1946,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidEncryptionParameters
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
+	case crypto.ErrInvalidEncryptionKeyID:
+		apiErr = ErrInvalidEncryptionKeyID
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
 	case crypto.ErrMissingCustomerKey:
@@ -2021,6 +2065,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSignatureDoesNotMatch
 	case hash.SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
+	case hash.ChecksumMismatch:
+		apiErr = ErrContentChecksumMismatch
 	case ObjectTooLarge:
 		apiErr = ErrEntityTooLarge
 	case ObjectTooSmall:
@@ -2049,7 +2095,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrRemoteDestinationNotFoundError
 	case BucketRemoteTargetNotFound:
 		apiErr = ErrRemoteTargetNotFoundError
-	case BucketRemoteConnectionErr:
+	case RemoteTargetConnectionErr:
 		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists
@@ -2226,7 +2272,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case crypto.Error:
 			apiErr = APIError{
-				Code:           "XMinIOEncryptionError",
+				Code:           "XMinioEncryptionError",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/api-headers.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/json"
-	"encoding/xml"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -30,6 +29,8 @@ import (
 
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	xxml "github.com/minio/xxml"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -64,9 +65,13 @@ func setCommonHeaders(w http.ResponseWriter) {
 // Encodes the response headers into XML format.
 func encodeResponse(response interface{}) []byte {
 	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
+	bytesBuffer.WriteString(xxml.Header)
+	buf, err := xxml.Marshal(response)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	bytesBuffer.Write(buf)
 	return bytesBuffer.Bytes()
 }
 
@@ -199,5 +204,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 	}
 
+	if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
+		if i := strings.LastIndexByte(v, '/'); i >= 0 {
+			v = v[i+1:]
+		}
+		w.Header()[xhttp.MinIOCompressed] = []string{v}
+	}
+
 	return nil
 }

cmd/api-response.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/handlers"
+	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 )
@@ -89,7 +90,8 @@ type ListVersionsResponse struct {
 	IsTruncated bool
 
 	CommonPrefixes []CommonPrefix
-	Versions       []ObjectVersion
+	DeleteMarkers  []DeleteMarkerVersion `xml:"DeleteMarker,omitempty"`
+	Versions       []ObjectVersion       `xml:"Version,omitempty"`
 
 	// Encoding type used to encode object keys in the response.
 	EncodingType string `xml:"EncodingType,omitempty"`
@@ -162,6 +164,12 @@ type Part struct {
 	LastModified string
 	ETag         string
 	Size         int64
+
+	// Checksum values
+	ChecksumCRC32  string
+	ChecksumCRC32C string
+	ChecksumSHA1   string
+	ChecksumSHA256 string
 }
 
 // ListPartsResponse - format for list parts response.
@@ -183,6 +191,7 @@ type ListPartsResponse struct {
 	MaxParts             int
 	IsTruncated          bool
 
+	ChecksumAlgorithm string
 	// List of parts.
 	Parts []Part `xml:"Part"`
 }
@@ -247,50 +256,80 @@ type ObjectVersion struct {
 	Object
 	IsLatest  bool
 	VersionID string `xml:"VersionId"`
-
-	isDeleteMarker bool
 }
 
-// MarshalXML - marshal ObjectVersion
-func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	if o.isDeleteMarker {
-		start.Name.Local = "DeleteMarker"
-	} else {
-		start.Name.Local = "Version"
+// DeleteMarkerVersion container for delete marker metadata
+type DeleteMarkerVersion struct {
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+
+	// Owner of the object.
+	Owner Owner
+
+	IsLatest  bool
+	VersionID string `xml:"VersionId"`
+}
+
+// Metadata metadata items implemented to ensure XML marshaling works.
+type Metadata struct {
+	Items []struct {
+		Key   string
+		Value string
 	}
-
-	type objectVersionWrapper ObjectVersion
-	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
-// StringMap is a map[string]string
-type StringMap map[string]string
+// Set add items, duplicate items get replaced.
+func (s *Metadata) Set(k, v string) {
+	for i, item := range s.Items {
+		if item.Key == k {
+			s.Items[i] = struct {
+				Key   string
+				Value string
+			}{
+				Key:   k,
+				Value: v,
+			}
+			return
+		}
+	}
+	s.Items = append(s.Items, struct {
+		Key   string
+		Value string
+	}{
+		Key:   k,
+		Value: v,
+	})
+}
+
+type xmlKeyEntry struct {
+	XMLName xml.Name
+	Value   string `xml:",chardata"`
+}
 
 // MarshalXML - StringMap marshals into XML.
-func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	tokens := []xml.Token{start}
-
-	for key, value := range s {
-		t := xml.StartElement{}
-		t.Name = xml.Name{
-			Space: "",
-			Local: key,
-		}
-		tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
+func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if s == nil {
+		return nil
 	}
 
-	tokens = append(tokens, xml.EndElement{
-		Name: start.Name,
-	})
+	if len(s.Items) == 0 {
+		return nil
+	}
 
-	for _, t := range tokens {
-		if err := e.EncodeToken(t); err != nil {
+	if err := e.EncodeToken(start); err != nil {
+		return err
+	}
+
+	for _, item := range s.Items {
+		if err := e.Encode(xmlKeyEntry{
+			XMLName: xml.Name{Local: item.Key},
+			Value:   item.Value,
+		}); err != nil {
 			return err
 		}
 	}
 
-	// flush to ensure tokens are written
-	return e.Flush()
+	return e.EncodeToken(start.End())
 }
 
 // Object container for object metadata
@@ -307,7 +346,7 @@ type Object struct {
 	StorageClass string
 
 	// UserMetadata user-defined metadata
-	UserMetadata StringMap `xml:"UserMetadata,omitempty"`
+	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -350,6 +389,11 @@ type CompleteMultipartUploadResponse struct {
 	Bucket   string
 	Key      string
 	ETag     string
+
+	ChecksumCRC32  string
+	ChecksumCRC32C string
+	ChecksumSHA1   string
+	ChecksumSHA256 string
 }
 
 // DeleteError structure.
@@ -438,6 +482,8 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
+	deleteMarkers := make([]DeleteMarkerVersion, 0, len(resp.Objects))
+
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
@@ -445,10 +491,25 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 	data := ListVersionsResponse{}
 
 	for _, object := range resp.Objects {
-		content := ObjectVersion{}
 		if object.Name == "" {
 			continue
 		}
+
+		if object.DeleteMarker {
+			deleteMarker := DeleteMarkerVersion{
+				Key:          s3EncodeName(object.Name, encodingType),
+				LastModified: object.ModTime.UTC().Format(iso8601TimeFormat),
+				Owner:        owner,
+				VersionID:    object.VersionID,
+			}
+			if deleteMarker.VersionID == "" {
+				deleteMarker.VersionID = nullVersionID
+			}
+			deleteMarker.IsLatest = object.IsLatest
+			deleteMarkers = append(deleteMarkers, deleteMarker)
+			continue
+		}
+		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
@@ -466,12 +527,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 			content.VersionID = nullVersionID
 		}
 		content.IsLatest = object.IsLatest
-		content.isDeleteMarker = object.DeleteMarker
 		versions = append(versions, content)
 	}
 
 	data.Name = bucket
 	data.Versions = versions
+	data.DeleteMarkers = deleteMarkers
 	data.EncodingType = encodingType
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.KeyMarker = s3EncodeName(marker, encodingType)
@@ -569,14 +630,14 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		}
 		content.Owner = owner
 		if metadata {
-			content.UserMetadata = make(StringMap)
+			content.UserMetadata = &Metadata{}
 			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
 			case crypto.S3:
-				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 			case crypto.S3KMS:
-				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
 			case crypto.SSEC:
-				content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
@@ -588,7 +649,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 					continue
 				}
-				content.UserMetadata[k] = v
+				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)
@@ -642,14 +703,20 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 }
 
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
-func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
-	return CompleteMultipartUploadResponse{
+func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
+	cs := oi.decryptChecksums()
+	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,
 		Key:      key,
 		// AWS S3 quotes the ETag in XML, make sure we are compatible here.
-		ETag: "\"" + etag + "\"",
+		ETag:           "\"" + oi.ETag + "\"",
+		ChecksumSHA1:   cs[hash.ChecksumSHA1.String()],
+		ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
+		ChecksumCRC32:  cs[hash.ChecksumCRC32.String()],
+		ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
 	}
+	return c
 }
 
 // generates ListPartsResponse from ListPartsInfo.
@@ -674,6 +741,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
 	listPartsResponse.IsTruncated = partsInfo.IsTruncated
 	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
+	listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
 
 	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
 	for index, part := range partsInfo.Parts {
@@ -682,6 +750,10 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
 		newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
+		newPart.ChecksumCRC32 = part.ChecksumCRC32
+		newPart.ChecksumCRC32C = part.ChecksumCRC32C
+		newPart.ChecksumSHA1 = part.ChecksumSHA1
+		newPart.ChecksumSHA256 = part.ChecksumSHA256
 		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse

cmd/api-router.go

@@ -342,7 +342,7 @@ func registerAPIRouter(router *mux.Router) {
 			collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "")
 		// ListenNotification
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
+			collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
 		// ResetBucketReplicationStatus - MinIO extension API
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "")
@@ -474,7 +474,7 @@ func registerAPIRouter(router *mux.Router) {
 
 	// ListenNotification
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
+		collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
 
 	// ListBuckets
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(

cmd/api-utils.go

@@ -35,8 +35,8 @@ func shouldEscape(c byte) bool {
 
 // s3URLEncode is based on Golang's url.QueryEscape() code,
 // while considering some S3 exceptions:
-//	- Avoid encoding '/' and '*'
-//	- Force encoding of '~'
+//   - Avoid encoding '/' and '*'
+//   - Force encoding of '~'
 func s3URLEncode(s string) string {
 	spaceCount, hexCount := 0, 0
 	for i := 0; i < len(s); i++ {

cmd/auth-handler.go

@@ -25,7 +25,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -85,6 +84,8 @@ func isRequestSignStreamingV4(r *http.Request) bool {
 }
 
 // Authorization type.
+//
+//go:generate stringer -type=authType -trimprefix=authType $GOFILE
 type authType int
 
 // List of all supported auth types.
@@ -217,8 +218,8 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		}
 	}
 
-	// If OPA is set, return without any further checks.
-	if globalPolicyOPA != nil {
+	// If AuthZPlugin is set, return without any further checks.
+	if newGlobalAuthZPluginFn() != nil {
 		return claims.Map(), nil
 	}
 
@@ -288,28 +289,33 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 }
 
 // Check request auth type verifies the incoming http request
-// - validates the request signature
-// - validates the policy action if anonymous tests bucket policies if any,
-//   for authenticated requests validates IAM policies.
+//   - validates the request signature
+//   - validates the policy action if anonymous tests bucket policies if any,
+//     for authenticated requests validates IAM policies.
+//
 // returns APIErrorCode if any to be replied to the client.
 func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
-	_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action, bucketName, objectName)
+	logger.GetReqInfo(ctx).BucketName = bucketName
+	logger.GetReqInfo(ctx).ObjectName = objectName
+
+	_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
 	return s3Err
 }
 
-// Check request auth type verifies the incoming http request
-// - validates the request signature
-// - validates the policy action if anonymous tests bucket policies if any,
-//   for authenticated requests validates IAM policies.
-// returns APIErrorCode if any to be replied to the client.
-// Additionally returns the accessKey used in the request, and if this request is by an admin.
-func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
+func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
+	if logger.GetReqInfo(ctx) == nil {
+		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
+		return ErrAccessDenied
+	}
+
+	var cred auth.Credentials
+	var owner bool
 	switch getRequestAuthType(r) {
 	case authTypeUnknown, authTypeStreamingSigned:
-		return cred, owner, ErrSignatureVersionNotSupported
+		return ErrSignatureVersionNotSupported
 	case authTypePresignedV2, authTypeSignedV2:
 		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
-			return cred, owner, s3Err
+			return s3Err
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
@@ -319,52 +325,67 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 			region = ""
 		}
 		if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
-			return cred, owner, s3Err
+			return s3Err
 		}
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
-		return cred, owner, s3Err
+		return s3Err
 	}
 
-	// LocationConstraint is valid only for CreateBucketAction.
-	var locationConstraint string
+	logger.GetReqInfo(ctx).Cred = cred
+	logger.GetReqInfo(ctx).Owner = owner
+
+	// region is valid only for CreateBucketAction.
+	var region string
 	if action == policy.CreateBucketAction {
 		// To extract region from XML in request body, get copy of request body.
-		payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
+		payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
 			logger.LogIf(ctx, err, logger.Application)
-			return cred, owner, ErrMalformedXML
+			return ErrMalformedXML
 		}
 
 		// Populate payload to extract location constraint.
-		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-
-		var s3Error APIErrorCode
-		locationConstraint, s3Error = parseLocationConstraint(r)
-		if s3Error != ErrNone {
-			return cred, owner, s3Error
+		r.Body = io.NopCloser(bytes.NewReader(payload))
+		region, s3Err = parseLocationConstraint(r)
+		if s3Err != ErrNone {
+			return s3Err
 		}
 
 		// Populate payload again to handle it in HTTP handler.
-		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-	}
-	if cred.AccessKey != "" {
-		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+		r.Body = io.NopCloser(bytes.NewReader(payload))
 	}
 
+	logger.GetReqInfo(ctx).Region = region
+
+	return s3Err
+}
+
+func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
+	reqInfo := logger.GetReqInfo(ctx)
+	if reqInfo == nil {
+		return ErrAccessDenied
+	}
+
+	cred := reqInfo.Cred
+	owner := reqInfo.Owner
+	region := reqInfo.Region
+	bucket := reqInfo.BucketName
+	object := reqInfo.ObjectName
+
 	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
-		// Anonymous checks are not meant for ListBuckets action
+		// Anonymous checks are not meant for ListAllBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Action:          action,
-			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, locationConstraint, "", nil),
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, region, "", nil),
 			IsOwner:         false,
-			ObjectName:      objectName,
+			ObjectName:      object,
 		}) {
 			// Request is allowed return the appropriate access key.
-			return cred, owner, ErrNone
+			return ErrNone
 		}
 
 		if action == policy.ListBucketVersionsAction {
@@ -373,31 +394,31 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 			if globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
 				Action:          policy.ListBucketAction,
-				BucketName:      bucketName,
-				ConditionValues: getConditionValues(r, locationConstraint, "", nil),
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, region, "", nil),
 				IsOwner:         false,
-				ObjectName:      objectName,
+				ObjectName:      object,
 			}) {
 				// Request is allowed return the appropriate access key.
-				return cred, owner, ErrNone
+				return ErrNone
 			}
 		}
 
-		return cred, owner, ErrAccessDenied
+		return ErrAccessDenied
 	}
 
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.Action(action),
-		BucketName:      bucketName,
+		BucketName:      bucket,
 		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
-		ObjectName:      objectName,
+		ObjectName:      object,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
 	}) {
 		// Request is allowed return the appropriate access key.
-		return cred, owner, ErrNone
+		return ErrNone
 	}
 
 	if action == policy.ListBucketVersionsAction {
@@ -407,18 +428,41 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
 			Action:          iampolicy.ListBucketAction,
-			BucketName:      bucketName,
+			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
-			ObjectName:      objectName,
+			ObjectName:      object,
 			IsOwner:         owner,
 			Claims:          cred.Claims,
 		}) {
 			// Request is allowed return the appropriate access key.
-			return cred, owner, ErrNone
+			return ErrNone
 		}
 	}
 
-	return cred, owner, ErrAccessDenied
+	return ErrAccessDenied
+}
+
+// Check request auth type verifies the incoming http request
+//   - validates the request signature
+//   - validates the policy action if anonymous tests bucket policies if any,
+//     for authenticated requests validates IAM policies.
+//
+// returns APIErrorCode if any to be replied to the client.
+// Additionally returns the accessKey used in the request, and if this request is by an admin.
+func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
+	s3Err = authenticateRequest(ctx, r, action)
+	reqInfo := logger.GetReqInfo(ctx)
+	if reqInfo == nil {
+		return cred, owner, ErrAccessDenied
+	}
+
+	cred = reqInfo.Cred
+	owner = reqInfo.Owner
+	if s3Err != ErrNone {
+		return cred, owner, s3Err
+	}
+
+	return cred, owner, authorizeRequest(ctx, r, action)
 }
 
 // Verify if request has valid AWS Signature Version '2'.
@@ -500,11 +544,18 @@ func isSupportedS3AuthType(aType authType) bool {
 func setAuthHandler(h http.Handler) http.Handler {
 	// handler for validating incoming authorization headers.
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		aType := getRequestAuthType(r)
 		if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
 			// Verify if date headers are set, if not reject the request
 			amzDate, errCode := parseAmzDateHeader(r)
 			if errCode != ErrNone {
+				if ok {
+					tc.funcName = "handler.Auth"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
@@ -516,6 +567,11 @@ func setAuthHandler(h http.Handler) http.Handler {
 			// or in the future, reject request otherwise.
 			curTime := UTCNow()
 			if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
+				if ok {
+					tc.funcName = "handler.Auth"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -525,6 +581,12 @@ func setAuthHandler(h http.Handler) http.Handler {
 			h.ServeHTTP(w, r)
 			return
 		}
+
+		if ok {
+			tc.funcName = "handler.Auth"
+			tc.responseRecorder.LogErrBody = true
+		}
+
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})
@@ -604,22 +666,22 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
+	region := globalSite.Region
 	switch atype {
 	case authTypeUnknown:
 		return ErrSignatureVersionNotSupported
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
-		region := globalSite.Region
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		return s3Err
 	}
 
-	if cred.AccessKey != "" {
-		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
-	}
+	logger.GetReqInfo(ctx).Cred = cred
+	logger.GetReqInfo(ctx).Owner = owner
+	logger.GetReqInfo(ctx).Region = region
 
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.

cmd/auth-handler_test.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -32,13 +31,19 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
+type nullReader struct{}
+
+func (r *nullReader) Read(b []byte) (int, error) {
+	return len(b), nil
+}
+
 // Test get request auth type.
 func TestGetRequestAuthType(t *testing.T) {
 	type testCase struct {
 		req   *http.Request
 		authT authType
 	}
-	nopCloser := ioutil.NopCloser(io.LimitReader(&nullReader{}, 1024))
+	nopCloser := io.NopCloser(io.LimitReader(&nullReader{}, 1024))
 	testCases := []testCase{
 		// Test case - 1
 		// Check for generic signature v4 header.
@@ -400,7 +405,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 	for i, testCase := range testCases {
 		s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
 		if s3Error != testCase.s3Error {
-			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
+			if _, err := io.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
 			}
 		}

cmd/authtype_string.go

@@ -0,0 +1,32 @@
+// Code generated by "stringer -type=authType -trimprefix=authType auth-handler.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[authTypeUnknown-0]
+	_ = x[authTypeAnonymous-1]
+	_ = x[authTypePresigned-2]
+	_ = x[authTypePresignedV2-3]
+	_ = x[authTypePostPolicy-4]
+	_ = x[authTypeStreamingSigned-5]
+	_ = x[authTypeSigned-6]
+	_ = x[authTypeSignedV2-7]
+	_ = x[authTypeJWT-8]
+	_ = x[authTypeSTS-9]
+}
+
+const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
+
+var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
+
+func (i authType) String() string {
+	if i < 0 || i >= authType(len(_authType_index)-1) {
+		return "authType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _authType_name[_authType_index[i]:_authType_index[i+1]]
+}

cmd/background-heal-ops.go

@@ -22,12 +22,14 @@ import (
 	"runtime"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/pubsub"
 )
 
 // healTask represents what to heal along with options
-//   path: '/' =>  Heal disk formats along with metadata
-//   path: 'bucket/' or '/bucket/' => Heal bucket
-//   path: 'bucket/object' => Heal object
+//
+//	path: '/' =>  Heal disk formats along with metadata
+//	path: 'bucket/' or '/bucket/' => Heal bucket
+//	path: 'bucket/object' => Heal object
 type healTask struct {
 	bucket    string
 	object    string
@@ -49,10 +51,10 @@ type healRoutine struct {
 	workers int
 }
 
-func systemIO() int {
+func activeListeners() int {
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	return int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
+	return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
 }
 
 func waitForLowHTTPReq() {
@@ -61,7 +63,7 @@ func waitForLowHTTPReq() {
 		currentIO = httpServer.GetRequestCount
 	}
 
-	globalHealConfig.Wait(currentIO, systemIO)
+	globalHealConfig.Wait(currentIO, activeListeners)
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {

cmd/background-newdisks-heal-ops.go

@@ -26,15 +26,12 @@ import (
 	"os"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/console"
 )
 
 const (
@@ -90,14 +87,14 @@ type healingTracker struct {
 // The disk ID will be validated against the loaded one.
 func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) {
 	if disk == nil {
-		return nil, errors.New("loadHealingTracker: nil disk given")
+		return nil, errors.New("loadHealingTracker: nil drive given")
 	}
 	diskID, err := disk.GetDiskID()
 	if err != nil {
 		return nil, err
 	}
 	b, err := disk.ReadAll(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename))
+		pathJoin(bucketMetaPrefix, healingTrackerFilename))
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +104,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 		return nil, err
 	}
 	if h.ID != diskID && h.ID != "" {
-		return nil, fmt.Errorf("loadHealingTracker: disk id mismatch expected %s, got %s", h.ID, diskID)
+		return nil, fmt.Errorf("loadHealingTracker: drive id mismatch expected %s, got %s", h.ID, diskID)
 	}
 	h.disk = disk
 	h.ID = diskID
@@ -132,7 +129,7 @@ func newHealingTracker(disk StorageAPI) *healingTracker {
 // If the tracker has been deleted an error is returned.
 func (h *healingTracker) update(ctx context.Context) error {
 	if h.disk.Healing() == nil {
-		return fmt.Errorf("healingTracker: disk %q is not marked as healing", h.ID)
+		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
 	}
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
@@ -158,15 +155,19 @@ func (h *healingTracker) save(ctx context.Context) error {
 	}
 	globalBackgroundHealState.updateHealStatus(h)
 	return h.disk.WriteAll(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+		pathJoin(bucketMetaPrefix, healingTrackerFilename),
 		htrackerBytes)
 }
 
 // delete the tracker on disk.
 func (h *healingTracker) delete(ctx context.Context) error {
 	return h.disk.Delete(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
-		false)
+		pathJoin(bucketMetaPrefix, healingTrackerFilename),
+		DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		},
+	)
 }
 
 func (h *healingTracker) isHealed(bucket string) bool {
@@ -258,26 +259,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	bgSeq := mustGetHealSequence(ctx)
-
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
-	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
-		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content - use 'mc admin heal alias/ --verbose' to check the status",
-			drivesToHeal, defaultMonitorNewDiskInterval))
-
-		// Heal any disk format and metadata early, if possible.
-		// Start with format healing
-		if err := bgSeq.healDiskFormat(); err != nil {
-			if newObjectLayerFn() != nil {
-				// log only in situations, when object layer
-				// has fully initialized.
-				logger.LogIf(bgSeq.ctx, err)
-			}
-		}
-	}
-
-	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
+	go monitorLocalDisksAndHeal(ctx, z)
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
@@ -299,10 +283,111 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	return disksToHeal
 }
 
+var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+
+func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
+	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
+
+	disk, format, err := connectEndpoint(endpoint)
+	if err != nil {
+		return fmt.Errorf("Error: %w, %s", err, endpoint)
+	}
+
+	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
+	if poolIdx < 0 {
+		return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
+	}
+
+	// Calculate the set index where the current endpoint belongs
+	z.serverPools[poolIdx].erasureDisksMu.RLock()
+	setIdx, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
+	z.serverPools[poolIdx].erasureDisksMu.RUnlock()
+	if err != nil {
+		return err
+	}
+	if setIdx < 0 {
+		return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
+	}
+
+	// Prevent parallel erasure set healing
+	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
+	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer locker.Unlock(lkctx.Cancel)
+
+	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
+	// Buckets data are dispersed in multiple zones/sets, make
+	// sure to heal all bucket metadata configuration.
+	buckets = append(buckets, BucketInfo{
+		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+	}, BucketInfo{
+		Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+	})
+
+	// Heal latest buckets first.
+	sort.Slice(buckets, func(i, j int) bool {
+		a, b := strings.HasPrefix(buckets[i].Name, minioMetaBucket), strings.HasPrefix(buckets[j].Name, minioMetaBucket)
+		if a != b {
+			return a
+		}
+		return buckets[i].Created.After(buckets[j].Created)
+	})
+
+	if serverDebugLog {
+		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
+	}
+
+	// Load healing tracker in this disk
+	tracker, err := loadHealingTracker(ctx, disk)
+	if err != nil {
+		// So someone changed the drives underneath, healing tracker missing.
+		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
+			disk, humanize.Ordinal(poolIdx+1), err))
+		tracker = newHealingTracker(disk)
+	}
+
+	// Load bucket totals
+	cache := dataUsageCache{}
+	if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil {
+		dataUsageInfo := cache.dui(dataUsageRoot, nil)
+		tracker.ObjectsTotalCount = dataUsageInfo.ObjectsTotalCount
+		tracker.ObjectsTotalSize = dataUsageInfo.ObjectsTotalSize
+	}
+
+	tracker.PoolIndex, tracker.SetIndex, tracker.DiskIndex = disk.GetDiskLoc()
+	tracker.setQueuedBuckets(buckets)
+	if err := tracker.save(ctx); err != nil {
+		return err
+	}
+
+	// Start or resume healing of this erasure set
+	if err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker); err != nil {
+		return err
+	}
+
+	if tracker.ItemsFailed > 0 {
+		logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	} else {
+		logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	}
+
+	if serverDebugLog {
+		tracker.printTo(os.Stdout)
+		logger.Info("\n")
+	}
+
+	logger.LogIf(ctx, tracker.delete(ctx))
+
+	return nil
+}
+
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval)
 	defer diskCheckTimer.Stop()
@@ -312,142 +397,37 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 		case <-ctx.Done():
 			return
 		case <-diskCheckTimer.C:
-			// Reset to next interval.
-			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
-
-			var erasureSetInPoolDisksToHeal []map[int][]StorageAPI
-
 			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
-			if len(healDisks) > 0 {
-				// Reformat disks
-				bgSeq.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
-
-				// Ensure that reformatting disks is finished
-				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)
-
-				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal - 'mc admin heal alias/ --verbose' to check the status.",
-					len(healDisks)))
-
-				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
-				for i := range z.serverPools {
-					erasureSetInPoolDisksToHeal[i] = map[int][]StorageAPI{}
-				}
+			if len(healDisks) == 0 {
+				// Reset for next interval.
+				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
+				continue
 			}
 
-			if serverDebugLog && len(healDisks) > 0 {
-				console.Debugf(color.Green("healDisk:")+" disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+			// Reformat disks immediately
+			_, err := z.HealFormat(context.Background(), false)
+			if err != nil && !errors.Is(err, errNoHealRequired) {
+				logger.LogIf(ctx, err)
+				// Reset for next interval.
+				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
+				continue
 			}
 
-			// heal only if new disks found.
-			for _, endpoint := range healDisks {
-				disk, format, err := connectEndpoint(endpoint)
-				if err != nil {
-					printEndpointError(endpoint, err, true)
-					continue
-				}
-
-				poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
-				if poolIdx < 0 {
-					continue
-				}
-
-				// Calculate the set index where the current endpoint belongs
-				z.serverPools[poolIdx].erasureDisksMu.RLock()
-				// Protect reading reference format.
-				setIndex, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
-				z.serverPools[poolIdx].erasureDisksMu.RUnlock()
-				if err != nil {
-					printEndpointError(endpoint, err, false)
-					continue
-				}
-
-				erasureSetInPoolDisksToHeal[poolIdx][setIndex] = append(erasureSetInPoolDisksToHeal[poolIdx][setIndex], disk)
-			}
-
-			buckets, _ := z.ListBuckets(ctx)
-
-			// Buckets data are dispersed in multiple zones/sets, make
-			// sure to heal all bucket metadata configuration.
-			buckets = append(buckets, BucketInfo{
-				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			}, BucketInfo{
-				Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
-			})
-
-			// Heal latest buckets first.
-			sort.Slice(buckets, func(i, j int) bool {
-				a, b := strings.HasPrefix(buckets[i].Name, minioMetaBucket), strings.HasPrefix(buckets[j].Name, minioMetaBucket)
-				if a != b {
-					return a
-				}
-				return buckets[i].Created.After(buckets[j].Created)
-			})
-
-			// TODO(klauspost): This will block until all heals are done,
-			// in the future this should be able to start healing other sets at once.
-			var wg sync.WaitGroup
-			for i, setMap := range erasureSetInPoolDisksToHeal {
-				i := i
-				for setIndex, disks := range setMap {
-					if len(disks) == 0 {
-						continue
+			for _, disk := range healDisks {
+				go func(disk Endpoint) {
+					globalBackgroundHealState.markDiskForHealing(disk)
+					err := healFreshDisk(ctx, z, disk)
+					if err != nil {
+						printEndpointError(disk, err, false)
+						return
 					}
-					wg.Add(1)
-					go func(setIndex int, disks []StorageAPI) {
-						defer wg.Done()
-						for _, disk := range disks {
-							if serverDebugLog {
-								logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
-							}
-
-							// So someone changed the drives underneath, healing tracker missing.
-							tracker, err := loadHealingTracker(ctx, disk)
-							if err != nil {
-								logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
-									disk, humanize.Ordinal(i+1), err))
-								tracker = newHealingTracker(disk)
-							}
-
-							// Load bucket totals
-							cache := dataUsageCache{}
-							if err := cache.load(ctx, z.serverPools[i].sets[setIndex], dataUsageCacheName); err == nil {
-								dataUsageInfo := cache.dui(dataUsageRoot, nil)
-								tracker.ObjectsTotalCount = dataUsageInfo.ObjectsTotalCount
-								tracker.ObjectsTotalSize = dataUsageInfo.ObjectsTotalSize
-							}
-
-							tracker.PoolIndex, tracker.SetIndex, tracker.DiskIndex = disk.GetDiskLoc()
-							tracker.setQueuedBuckets(buckets)
-							if err := tracker.save(ctx); err != nil {
-								logger.LogIf(ctx, err)
-								// Unable to write healing tracker, permission denied or some
-								// other unexpected error occurred. Proceed to look for new
-								// disks to be healed again, we cannot proceed further.
-								return
-							}
-
-							err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, tracker.QueuedBuckets, tracker)
-							if err != nil {
-								logger.LogIf(ctx, err)
-								continue
-							}
-
-							if serverDebugLog {
-								logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
-									humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
-								logger.Info("Summary:\n")
-								tracker.printTo(os.Stdout)
-								logger.Info("\n")
-							}
-							logger.LogIf(ctx, tracker.delete(ctx))
-
-							// Only upon success pop the healed disk.
-							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
-						}
-					}(setIndex, disks)
-				}
+					// Only upon success pop the healed disk.
+					globalBackgroundHealState.popHealLocalDisks(disk)
+				}(disk)
 			}
-			wg.Wait()
+
+			// Reset for next interval.
+			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 		}
 	}
 }

cmd/benchmark-utils_test.go

@@ -25,7 +25,7 @@ import (
 	"strconv"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 // Benchmark utility functions for ObjectLayer.PutObject().
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -85,12 +85,12 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 
 	// PutObjectPart returns etag of the object inserted.
 	// etag variable is assigned with that value.
-	var etag, uploadID string
+	var etag string
 	// get text data generated for number of bytes equal to object size.
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
+	res, err := obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -113,7 +113,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			}
 			md5hex := getMD5Hash(textPartData)
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
+			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, res.UploadID, j,
 				mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bitrot-streaming.go

@@ -152,10 +152,10 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		if len(b.data) == 0 && b.tillOffset != streamOffset {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
-			if err != nil {
+			if err != nil && err != errDiskNotFound {
 				logger.LogIf(GlobalContext,
-					fmt.Errorf("Error(%w) reading erasure shards at (%s: %s/%s), will attempt to reconstruct if we have quorum",
-						err, b.disk, b.volume, b.filePath))
+					fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
+						b.disk, b.volume, b.filePath, err))
 			}
 		} else {
 			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
@@ -180,7 +180,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s  -> %s/%s - content hash does not match - expected %s, got %s",
 			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
 		return 0, errFileCorrupt
 	}

cmd/bitrot-whole.go

@@ -38,12 +38,12 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -72,12 +72,12 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
+			logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -27,6 +26,7 @@ import (
 	"io"
 
 	"github.com/minio/highwayhash"
+	"github.com/minio/minio/internal/hash/sha256"
 	"golang.org/x/crypto/blake2b"
 
 	xioutil "github.com/minio/minio/internal/ioutil"

cmd/bitrot_test.go

@@ -20,17 +20,11 @@ package cmd
 import (
 	"context"
 	"io"
-	"io/ioutil"
-	"os"
 	"testing"
 )
 
 func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
-	tmpDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	volume := "testvol"
 	filePath := "testfile"
@@ -60,7 +54,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	writer.(io.Closer).Close()
+	if bw, ok := writer.(io.Closer); ok {
+		bw.Close()
+	}
 
 	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
@@ -76,6 +72,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if _, err = reader.ReadAt(b[:5], 30); err != nil {
 		t.Fatal(err)
 	}
+	if br, ok := reader.(io.Closer); ok {
+		br.Close()
+	}
 }
 
 func TestAllBitrotAlgorithms(t *testing.T) {

cmd/bucket-encryption-handlers.go

@@ -65,7 +65,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	kmsKey := encConfig.KeyID()
 	if kmsKey != "" {
 		kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
-		_, err := GlobalKMS.GenerateKey(kmsKey, kmsContext)
+		_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
 		if err != nil {
 			if errors.Is(err, kes.ErrKeyNotFound) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errKMSKeyNotFound), r.URL)
@@ -108,7 +108,8 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Store the bucket encryption configuration in the object layer
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -122,6 +123,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -153,7 +155,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -196,13 +198,14 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Delete bucket encryption config from object layer
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -212,6 +215,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-handlers.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
-	"encoding/json"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -33,7 +32,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
@@ -69,11 +67,16 @@ const (
 // - Check if a bucket has an entry in etcd backend
 // -- If no, make an entry
 // -- If yes, check if the entry matches local IP check if we
-//    need to update the entry then proceed to update
+//
+//	need to update the entry then proceed to update
+//
 // -- If yes, check if the IP of entry matches local IP.
-//    This means entry is for this instance.
+//
+//	This means entry is for this instance.
+//
 // -- If IP of the entry doesn't match, this means entry is
-//    for another instance. Log an error to console.
+//
+//	for another instance. Log an error to console.
 func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	if len(buckets) == 0 {
 		return
@@ -203,7 +206,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -229,7 +232,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 // using the Initiate Multipart Upload request, but has not yet been
 // completed or aborted. This operation returns at most 1,000 multipart
 // uploads in the response.
-//
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
@@ -298,7 +300,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 	listBuckets := objectAPI.ListBuckets
 
-	cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction, "", "")
+	cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction)
 	if s3Error != ErrNone && s3Error != ErrAccessDenied {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -334,7 +336,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	} else {
 		// Invoke the list buckets.
 		var err error
-		bucketsInfo, err = listBuckets(ctx)
+		bucketsInfo, err = listBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -364,6 +366,18 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			}) {
 				bucketsInfo[n] = bucketInfo
 				n++
+			} else if globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
+				Action:          iampolicy.GetBucketLocationAction,
+				BucketName:      bucketInfo.Name,
+				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				IsOwner:         owner,
+				ObjectName:      "",
+				Claims:          cred.Claims,
+			}) {
+				bucketsInfo[n] = bucketInfo
+				n++
 			}
 		}
 		bucketsInfo = bucketsInfo[:n]
@@ -437,7 +451,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
 
 	// Before proceeding validate if bucket exists.
-	_, err := objectAPI.GetBucketInfo(ctx, bucket)
+	_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -471,9 +485,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		hasLockEnabled = true
 	}
 
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	suspended := globalBucketVersioningSys.Suspended(bucket)
-
 	type deleteResult struct {
 		delInfo DeletedObject
 		errInfo DeleteError
@@ -481,8 +492,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
 
+	vc, _ := globalBucketVersioningSys.Get(bucket)
 	oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
-
 	for index, object := range deleteObjectsReq.Objects {
 		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
 			if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
@@ -514,8 +525,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 		opts := ObjectOptions{
 			VersionID:        object.VersionID,
-			Versioned:        versioned,
-			VersionSuspended: suspended,
+			Versioned:        vc.PrefixEnabled(object.ObjectName),
+			VersionSuspended: vc.Suspended(),
 		}
 
 		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
@@ -526,7 +537,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if !globalTierConfigMgr.Empty() {
-			oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(opts.VersionID).WithVersioning(versioned, suspended)
+			oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
 			oss[index].SetTransitionState(goi.TransitionedObject)
 		}
 
@@ -581,8 +592,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		Versioned:        versioned,
-		VersionSuspended: suspended,
+		PrefixEnabledFn:  vc.PrefixEnabled,
+		VersionSuspended: vc.Suspended(),
 	})
 
 	for i := range errs {
@@ -638,22 +649,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
-		if replicateDeletes {
-			if dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending {
-				dv := DeletedObjectReplicationInfo{
-					DeletedObject: dobj,
-					Bucket:        bucket,
-				}
-				scheduleReplicationDelete(ctx, dv, objectAPI)
+		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
+			dv := DeletedObjectReplicationInfo{
+				DeletedObject: dobj,
+				Bucket:        bucket,
+				EventType:     ReplicateIncomingDelete,
 			}
-		}
-
-	}
-
-	// Notify deleted event for objects.
-	for _, dobj := range deletedObjects {
-		if dobj.ObjectName == "" {
-			continue
+			scheduleReplicationDelete(ctx, dv, objectAPI)
 		}
 
 		eventName := event.ObjectRemovedDelete
@@ -729,11 +731,30 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
+	cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction)
+	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 
+	if objectLockEnabled {
+		// Creating a bucket with locking requires the user having more permissions
+		for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
+				Action:          action,
+				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				BucketName:      bucket,
+				IsOwner:         owner,
+				Claims:          cred.Claims,
+			}) {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+				return
+			}
+		}
+	}
+
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
@@ -748,7 +769,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	opts := BucketOptions{
+	// check if client is attempting to create more buckets than allowed maximum.
+	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
+		apiErr := errorCodes.ToAPIErr(ErrTooManyBuckets)
+		apiErr.Description = fmt.Sprintf("You have attempted to create %d buckets than allowed %d", currBuckets+1, maxBuckets)
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+
+	opts := MakeBucketOptions{
 		Location:    location,
 		LockEnabled: objectLockEnabled,
 		ForceCreate: forceCreate,
@@ -767,7 +796,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+						Force:      false,
+						NoRecreate: true,
+						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+					})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -809,19 +842,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
-	if _, ok := err.(BucketExists); ok {
-		// Though bucket exists locally, we send the site-replication
-		// hook to ensure all sites have this bucket. If the hook
-		// succeeds, the client will still receive a bucket exists
-		// message.
-		err2 := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
-		if err2 != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+	if err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
+		if _, ok := err.(BucketExists); ok {
+			// Though bucket exists locally, we send the site-replication
+			// hook to ensure all sites have this bucket. If the hook
+			// succeeds, the client will still receive a bucket exists
+			// message.
+			globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
 		}
-	}
-	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -830,8 +858,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	err = globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
-	if err != nil {
+	if err := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -873,7 +900,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1049,7 +1076,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if crypto.Requested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
 				return
@@ -1075,7 +1102,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					return
 				}
 			}
-			reader, objectEncryptionKey, err = newEncryptReader(hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+			reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1124,9 +1151,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if successRedirect != "" {
-		// Replace raw query params..
-		redirectURL.RawQuery = getRedirectPostRawQuery(objInfo)
+	if redirectURL != nil { // success_action_redirect is valid and set.
+		v := redirectURL.Query()
+		v.Add("bucket", objInfo.Bucket)
+		v.Add("key", objInfo.Name)
+		v.Add("etag", "\""+objInfo.ETag+"\"")
+		redirectURL.RawQuery = v.Encode()
 		writeRedirectSeeOther(w, redirectURL.String())
 		return
 	}
@@ -1170,7 +1200,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1233,7 +1263,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
@@ -1310,7 +1340,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
+		Force:      forceDelete,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+	}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1365,7 +1398,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1394,7 +1427,8 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1408,6 +1442,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
+		UpdatedAt:        updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1475,7 +1510,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1499,7 +1534,8 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1510,9 +1546,10 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
 	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeTags,
-		Bucket: bucket,
-		Tags:   &cfgStr,
+		Type:      madmin.SRBucketMetaTypeTags,
+		Bucket:    bucket,
+		Tags:      &cfgStr,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1581,14 +1618,16 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeTags,
-		Bucket: bucket,
+		Type:      madmin.SRBucketMetaTypeTags,
+		Bucket:    bucket,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1597,375 +1636,3 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
-
-// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
-// ----------
-// Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
-func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "PutBucketReplicationConfig")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-	if !globalIsErasure {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if globalSiteReplicationSys.isEnabled() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
-		return
-	}
-	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
-		return
-	}
-	replicationConfig, err := replication.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
-		apiErr.Description = err.Error()
-		writeErrorResponse(ctx, w, apiErr, r.URL)
-		return
-	}
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig)
-	if apiErr != noError {
-		writeErrorResponse(ctx, w, apiErr, r.URL)
-		return
-	}
-	// Validate the received bucket replication config
-	if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	configData, err := xml.Marshal(replicationConfig)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
-}
-
-// GetBucketReplicationConfigHandler - GET Bucket replication configuration.
-// ----------
-// Gets the replication configuration for a bucket.
-func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketReplicationConfig")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	// check if user has permissions to perform this operation
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	configData, err := xml.Marshal(config)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseXML(w, configData)
-}
-
-// DeleteBucketReplicationConfigHandler - DELETE Bucket replication config.
-// ----------
-func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DeleteBucketReplicationConfig")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if globalSiteReplicationSys.isEnabled() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
-		return
-	}
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
-}
-
-// GetBucketReplicationMetricsHandler - GET Bucket replication metrics.
-// ----------
-// Gets the replication metrics for a bucket.
-func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketReplicationMetrics")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	// check if user has permissions to perform this operation
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	var usageInfo BucketUsageInfo
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
-		usageInfo = dataUsageInfo.BucketsUsage[bucket]
-	}
-
-	w.Header().Set(xhttp.ContentType, string(mimeJSON))
-
-	enc := json.NewEncoder(w)
-	if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-}
-
-// ResetBucketReplicationStartHandler - starts a replication reset for all objects in a bucket which
-// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
-// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
-// remote target is entirely lost,and previously replicated objects need to be re-synced. If resync is
-// already in progress it returns an error
-func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ResetBucketReplicationStart")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	durationStr := r.URL.Query().Get("older-than")
-	arn := r.URL.Query().Get("arn")
-	resetID := r.URL.Query().Get("reset-id")
-	if resetID == "" {
-		resetID = mustGetUUID()
-	}
-	var (
-		days time.Duration
-		err  error
-	)
-	if durationStr != "" {
-		days, err = time.ParseDuration(durationStr)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{
-				Bucket: bucket,
-				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
-			}), r.URL)
-		}
-	}
-	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if !config.HasActiveRules("", true) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoMatchingRuleError), r.URL)
-		return
-	}
-	tgtArns := config.FilterTargetArns(
-		replication.ObjectOpts{
-			OpType:    replication.ResyncReplicationType,
-			TargetArn: arn,
-		})
-
-	if len(tgtArns) == 0 {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
-			Bucket: bucket,
-			Err:    fmt.Errorf("Remote target ARN %s missing or ineligible for replication resync", arn),
-		}), r.URL)
-		return
-	}
-
-	if len(tgtArns) > 1 && arn == "" {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
-			Bucket: bucket,
-			Err:    fmt.Errorf("ARN should be specified for replication reset"),
-		}), r.URL)
-		return
-	}
-	var rinfo ResyncTargetsInfo
-	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0])
-	target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
-	target.ResetID = resetID
-	rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
-	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
-		switch err.(type) {
-		case BucketRemoteConnectionErr:
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
-		default:
-			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
-		}
-	}
-	if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
-			Bucket: bucket,
-			Err:    err,
-		}), r.URL)
-		return
-	}
-
-	data, err := json.Marshal(rinfo)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	// Write success response.
-	writeSuccessResponseJSON(w, data)
-}
-
-// ResetBucketReplicationStatusHandler - returns the status of replication reset.
-// This API is a MinIO only extension
-func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ResetBucketReplicationStatus")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	arn := r.URL.Query().Get("arn")
-	var err error
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	globalReplicationPool.resyncState.RLock()
-	brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
-	if !ok {
-		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
-		if err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
-				Bucket: bucket,
-				Err:    fmt.Errorf("No replication resync status available for %s", arn),
-			}), r.URL)
-		}
-		return
-	}
-
-	var rinfo ResyncTargetsInfo
-	for tarn, st := range brs.TargetsMap {
-		if arn != "" && tarn != arn {
-			continue
-		}
-		rinfo.Targets = append(rinfo.Targets, ResyncTarget{
-			Arn:             tarn,
-			ResetID:         st.ResyncID,
-			StartTime:       st.StartTime,
-			EndTime:         st.EndTime,
-			ResyncStatus:    st.ResyncStatus.String(),
-			ReplicatedSize:  st.ReplicatedSize,
-			ReplicatedCount: st.ReplicatedCount,
-			FailedSize:      st.FailedSize,
-			FailedCount:     st.FailedCount,
-			Bucket:          st.Bucket,
-			Object:          st.Object,
-		})
-	}
-	globalReplicationPool.resyncState.RUnlock()
-	data, err := json.Marshal(rinfo)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	// Write success response.
-	writeSuccessResponseJSON(w, data)
-}

cmd/bucket-handlers_test.go

@@ -21,7 +21,7 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -885,7 +885,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		}
 
 		// read the response body.
-		actualContent, err = ioutil.ReadAll(rec.Body)
+		actualContent, err = io.ReadAll(rec.Body)
 		if err != nil {
 			t.Fatalf("Test %d : MinIO %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}

cmd/bucket-lifecycle-handlers.go

@@ -62,12 +62,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
+	bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -91,7 +91,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -163,12 +163,12 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/event"
@@ -330,7 +331,7 @@ const (
 // 2. when a transitioned object expires (based on an ILM rule).
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
 	var opts ObjectOptions
-	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
+	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
 	opts.Expiration = ExpirationOptions{Expire: true}
 	switch action {
@@ -415,13 +416,43 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 			ETag:   oi.ETag,
 		},
 		VersionID:        oi.VersionID,
-		Versioned:        globalBucketVersioningSys.Enabled(oi.Bucket),
-		VersionSuspended: globalBucketVersioningSys.Suspended(oi.Bucket),
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		MTime:            oi.ModTime,
 	}
 	return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
 
+type auditTierOp struct {
+	Tier             string `json:"tier"`
+	TimeToResponseNS int64  `json:"timeToResponseNS"`
+	OutputBytes      int64  `json:"tx,omitempty"`
+	Error            string `json:"error,omitempty"`
+}
+
+func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) {
+	startTime := time.Now()
+	return func(err error) {
+		// Record only when audit targets configured.
+		if len(logger.AuditTargets()) == 0 {
+			return
+		}
+
+		op := auditTierOp{
+			Tier:        tier,
+			OutputBytes: bytes,
+		}
+
+		if err == nil {
+			op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
+		} else {
+			op.Error = err.Error()
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("tierStats", op)
+	}
+}
+
 // getTransitionedObjectReader returns a reader from the transitioned tier.
 func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	tgtClient, err := globalTierConfigMgr.getDriver(oi.TransitionedObject.Tier)
@@ -441,12 +472,13 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
 		gopts.length = length
 	}
 
+	timeTierAction := auditTierActions(ctx, oi.TransitionedObject.Tier, length)
 	reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts)
 	if err != nil {
 		return nil, err
 	}
 	closer := func() {
-		reader.Close()
+		timeTierAction(reader.Close())
 	}
 	return fn(reader, h, closer)
 }
@@ -560,7 +592,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 	}
 	// Check if bucket exists.
 	if !r.OutputLocation.IsEmpty() {
-		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
+		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil {
 			return err
 		}
 		if r.OutputLocation.S3.Prefix == "" {
@@ -575,8 +607,8 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 
 // postRestoreOpts returns ObjectOptions with version-id from the POST restore object request for a given bucket and object.
 func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
+	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 	vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
@@ -627,8 +659,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
-			Versioned:        globalBucketVersioningSys.Enabled(bucket),
-			VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+			Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
+			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
 			UserDefined:      meta,
 		}
 	}
@@ -640,8 +672,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	}
 
 	return ObjectOptions{
-		Versioned:        globalBucketVersioningSys.Enabled(bucket),
-		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
 		UserDefined:      meta,
 		VersionID:        objInfo.VersionID,
 		MTime:            objInfo.ModTime,
@@ -755,7 +787,7 @@ func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
 		if strings.TrimSpace(expiryTokens[0]) != "expiry-date" {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}
-		expiry, err := time.Parse(http.TimeFormat, strings.Trim(expiryTokens[1], `"`))
+		expiry, err := amztime.ParseHeader(strings.Trim(expiryTokens[1], `"`))
 		if err != nil {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}

cmd/bucket-lifecycle_test.go

@@ -203,7 +203,7 @@ func TestObjectIsRemote(t *testing.T) {
 		if got := fi.IsRemote(); got != tc.remote {
 			t.Fatalf("Test %d.a: expected %v got %v", i+1, tc.remote, got)
 		}
-		oi := fi.ToObjectInfo("bucket", "object")
+		oi := fi.ToObjectInfo("bucket", "object", false)
 		if got := oi.IsRemote(); got != tc.remote {
 			t.Fatalf("Test %d.b: expected %v got %v", i+1, tc.remote, got)
 		}

cmd/bucket-listobjects-handlers.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -33,9 +32,9 @@ import (
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
-// - delimiter if set should be equal to '/', otherwise the request is rejected.
-// - marker if set should have a common prefix with 'prefix' param, otherwise
-//   the request is rejected.
+//   - delimiter if set should be equal to '/', otherwise the request is rejected.
+//   - marker if set should have a common prefix with 'prefix' param, otherwise
+//     the request is rejected.
 func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
 	// Max keys cannot be negative.
 	if maxKeys < 0 {
@@ -101,7 +100,6 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectVersionsInfo.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -167,7 +165,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -246,7 +243,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -302,7 +298,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 // This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
-//
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
@@ -347,7 +342,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsInfo.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -45,6 +45,14 @@ type BucketMetadataSys struct {
 	metadataMap map[string]BucketMetadata
 }
 
+// Count returns number of bucket metadata map entries.
+func (sys *BucketMetadataSys) Count() int {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	return len(sys.metadataMap)
+}
+
 // Remove bucket metadata from memory.
 func (sys *BucketMetadataSys) Remove(bucket string) {
 	if globalIsGateway {
@@ -75,28 +83,28 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
-func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) error {
+func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if globalIsGateway && globalGatewayName != NASBackendGateway {
 		if configFile == bucketPolicyConfig {
 			if configData == nil {
-				return objAPI.DeleteBucketPolicy(ctx, bucket)
+				return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
 			}
 			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
 			if err != nil {
-				return err
+				return updatedAt, err
 			}
-			return objAPI.SetBucketPolicy(ctx, bucket, config)
+			return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
 		}
-		return NotImplemented{}
+		return updatedAt, NotImplemented{}
 	}
 
 	if bucket == minioMetaBucket {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
@@ -105,64 +113,56 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 			// Only single drive mode needs this fallback.
 			meta = newBucketMetadata(bucket)
 		} else {
-			return err
+			return updatedAt, err
 		}
 	}
-
+	updatedAt = UTCNow()
 	switch configFile {
 	case bucketPolicyConfig:
 		meta.PolicyConfigJSON = configData
-		meta.PolicyConfigUpdatedAt = UTCNow()
+		meta.PolicyConfigUpdatedAt = updatedAt
 	case bucketNotificationConfig:
 		meta.NotificationConfigXML = configData
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
-		meta.EncryptionConfigUpdatedAt = UTCNow()
+		meta.EncryptionConfigUpdatedAt = updatedAt
 	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
-		meta.TaggingConfigUpdatedAt = UTCNow()
+		meta.TaggingConfigUpdatedAt = updatedAt
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
-		meta.QuotaConfigUpdatedAt = UTCNow()
+		meta.QuotaConfigUpdatedAt = updatedAt
 	case objectLockConfig:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
-		}
 		meta.ObjectLockConfigXML = configData
-		meta.ObjectLockConfigUpdatedAt = UTCNow()
+		meta.ObjectLockConfigUpdatedAt = updatedAt
 	case bucketVersioningConfig:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
-		}
 		meta.VersioningConfigXML = configData
+		meta.VersioningConfigUpdatedAt = updatedAt
 	case bucketReplicationConfig:
-		if !globalIsErasure && !globalIsDistErasure {
-			return NotImplemented{}
-		}
 		meta.ReplicationConfigXML = configData
-		meta.ReplicationConfigUpdatedAt = UTCNow()
+		meta.ReplicationConfigUpdatedAt = updatedAt
 	case bucketTargetsFile:
-		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{
+		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
 			bucket:            meta.Name,
 			bucketTargetsFile: bucketTargetsFile,
 		})
 		if err != nil {
-			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+			return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
 	default:
-		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
+		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
 	if err := meta.Save(ctx, objAPI); err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.Set(bucket, meta)
 	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here
 
-	return nil
+	return updatedAt, nil
 }
 
 // Get metadata for a bucket.
@@ -193,12 +193,15 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 
 // GetVersioningConfig returns configured versioning config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
+func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, time.Time, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
-		return nil, err
+		if errors.Is(err, errConfigNotFound) {
+			return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, meta.Created, nil
+		}
+		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, time.Time{}, err
 	}
-	return meta.versioningConfig, nil
+	return meta.versioningConfig, meta.VersioningConfigUpdatedAt, nil
 }
 
 // GetTaggingConfig returns configured tagging config
@@ -315,26 +318,27 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
+func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerFn()
 		if objAPI == nil {
-			return nil, errServerNotInitialized
+			return nil, time.Time{}, errServerNotInitialized
 		}
-		return objAPI.GetBucketPolicy(GlobalContext, bucket)
+		p, err := objAPI.GetBucketPolicy(GlobalContext, bucket)
+		return p, UTCNow(), err
 	}
 
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketPolicyNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.policyConfig == nil {
-		return nil, BucketPolicyNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
 	}
-	return meta.policyConfig, nil
+	return meta.policyConfig, meta.PolicyConfigUpdatedAt, nil
 }
 
 // GetQuotaConfig returns configured bucket quota
@@ -342,6 +346,9 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, er
 func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
 	meta, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, time.Time{}, BucketQuotaConfigNotFound{Bucket: bucket}
+		}
 		return nil, time.Time{}, err
 	}
 	return meta.quotaConfig, meta.QuotaConfigUpdatedAt, nil
@@ -369,6 +376,9 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketRemoteTargetNotFound{Bucket: bucket}
+		}
 		return nil, err
 	}
 	if meta.bucketTargetConfig == nil {
@@ -377,20 +387,6 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	return meta.bucketTargetConfig, nil
 }
 
-// GetBucketTarget returns the target for the bucket and arn.
-func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
-	targets, err := sys.GetBucketTargetsConfig(bucket)
-	if err != nil {
-		return madmin.BucketTarget{}, err
-	}
-	for _, t := range targets.Targets {
-		if t.Arn == arn {
-			return t, nil
-		}
-	}
-	return madmin.BucketTarget{}, errConfigNotFound
-}
-
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (BucketMetadata, error) {
@@ -464,7 +460,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			sys.metadataMap[buckets[index].Name] = meta
 			sys.Unlock()
 
-			globalNotificationSys.set(buckets[index], meta) // set notification targets
+			globalEventNotifier.set(buckets[index], meta) // set notification targets
 
 			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
 

cmd/bucket-metadata.go

@@ -87,6 +87,7 @@ type BucketMetadata struct {
 	TaggingConfigUpdatedAt      time.Time
 	QuotaConfigUpdatedAt        time.Time
 	ReplicationConfigUpdatedAt  time.Time
+	VersioningConfigUpdatedAt   time.Time
 
 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.Policy
@@ -104,10 +105,8 @@ type BucketMetadata struct {
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
 func newBucketMetadata(name string) BucketMetadata {
-	now := UTCNow()
 	return BucketMetadata{
-		Name:    name,
-		Created: now,
+		Name: name,
 		notificationConfig: &event.Config{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
@@ -120,6 +119,17 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 
+// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
+// creation time of bucket on this cluster in all other cases.
+func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
+	if b.Created.IsZero() {
+		b.Created = UTCNow()
+	}
+	if !createdAt.IsZero() {
+		b.Created = createdAt.UTC()
+	}
+}
+
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
@@ -159,17 +169,19 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
-
+	if err == nil {
+		b.defaultTimestamps()
+	}
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
 	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
 		return b, err
 	}
 
 	// migrate unencrypted remote targets
-	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
+	if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
 		return b, err
 	}
-	b.defaultTimestamps()
+
 	return b, nil
 }
 
@@ -344,6 +356,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.BucketTargetsConfigJSON = configData
 		}
 	}
+	b.defaultTimestamps()
 
 	if err := b.Save(ctx, objectAPI); err != nil {
 		return err
@@ -364,6 +377,7 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.PolicyConfigUpdatedAt.IsZero() {
 		b.PolicyConfigUpdatedAt = b.Created
 	}
+
 	if b.EncryptionConfigUpdatedAt.IsZero() {
 		b.EncryptionConfigUpdatedAt = b.Created
 	}
@@ -383,6 +397,10 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.ReplicationConfigUpdatedAt.IsZero() {
 		b.ReplicationConfigUpdatedAt = b.Created
 	}
+
+	if b.VersioningConfigUpdatedAt.IsZero() {
+		b.VersioningConfigUpdatedAt = b.Created
+	}
 }
 
 // Save config to supplied ObjectLayer api.
@@ -432,7 +450,7 @@ func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI Obje
 		return nil
 	}
 
-	encBytes, metaBytes, err := encryptBucketMetadata(b.Name, b.BucketTargetsConfigJSON, kms.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
+	encBytes, metaBytes, err := encryptBucketMetadata(ctx, b.Name, b.BucketTargetsConfigJSON, kms.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
 	if err != nil {
 		return err
 	}
@@ -443,14 +461,14 @@ func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI Obje
 }
 
 // encrypt bucket metadata if kms is configured.
-func encryptBucketMetadata(bucket string, input []byte, kmsContext kms.Context) (output, metabytes []byte, err error) {
+func encryptBucketMetadata(ctx context.Context, bucket string, input []byte, kmsContext kms.Context) (output, metabytes []byte, err error) {
 	if GlobalKMS == nil {
 		output = input
 		return
 	}
 
 	metadata := make(map[string]string)
-	key, err := GlobalKMS.GenerateKey("", kmsContext)
+	key, err := GlobalKMS.GenerateKey(ctx, "", kmsContext)
 	if err != nil {
 		return
 	}
@@ -459,7 +477,7 @@ func encryptBucketMetadata(bucket string, input []byte, kmsContext kms.Context)
 	objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader)
 	sealedKey := objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
 	crypto.S3.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey)
-	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	if err != nil {
 		return output, metabytes, err
 	}
@@ -489,6 +507,6 @@ func decryptBucketMetadata(input []byte, bucket string, meta map[string]string,
 	}
 
 	outbuf := bytes.NewBuffer(nil)
-	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	return outbuf.Bytes(), err
 }

cmd/bucket-metadata_gen.go

@@ -144,6 +144,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
 				return
 			}
+		case "VersioningConfigUpdatedAt":
+			z.VersioningConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -157,9 +163,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 20
+	// map header, size 21
 	// write "Name"
-	err = en.Append(0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -358,15 +364,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
 		return
 	}
+	// write "VersioningConfigUpdatedAt"
+	err = en.Append(0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.VersioningConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 20
+	// map header, size 21
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x14, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -425,6 +441,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ReplicationConfigUpdatedAt"
 	o = append(o, 0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.ReplicationConfigUpdatedAt)
+	// string "VersioningConfigUpdatedAt"
+	o = append(o, 0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.VersioningConfigUpdatedAt)
 	return
 }
 
@@ -566,6 +585,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ReplicationConfigUpdatedAt")
 				return
 			}
+		case "VersioningConfigUpdatedAt":
+			z.VersioningConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -580,6 +605,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize
 	return
 }

cmd/bucket-notification-handlers.go

@@ -60,7 +60,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -72,7 +72,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 	config.SetRegion(globalSite.Region)
-	if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
+	if err = config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
 		if ok {
 			for i, queue := range config.QueueList {
@@ -132,7 +132,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -144,7 +144,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalEventNotifier.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
@@ -160,13 +160,13 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	rulesMap := config.ToRulesMap()
-	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
+	globalEventNotifier.AddRulesMap(bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/bucket-object-lock.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"math"
 	"net/http"
 
@@ -46,11 +47,14 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 
 	config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
 	if err != nil {
-		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
+		if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) {
 			return r, nil
 		}
+		if errors.Is(err, errInvalidArgument) {
+			return r, err
+		}
+		logger.CriticalIf(context.Background(), err)
 		return r, err
-
 	}
 	return config.ToRetention(), nil
 }

cmd/bucket-policy-handlers.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
@@ -61,7 +60,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -79,7 +78,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	bucketPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -103,16 +102,18 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Call site replication hook.
 	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypePolicy,
-		Bucket: bucket,
-		Policy: bucketPolicyBytes,
+		Type:      madmin.SRBucketMetaTypePolicy,
+		Bucket:    bucket,
+		Policy:    bucketPolicyBytes,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -143,20 +144,22 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Call site replication hook.
 	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypePolicy,
-		Bucket: bucket,
+		Type:      madmin.SRBucketMetaTypePolicy,
+		Bucket:    bucket,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -187,7 +190,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers_test.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -128,7 +127,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 			defer wg.Done()
 			// Sync start.
 			<-start
-			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					t.Logf("unexpected error: %T: %v", err, err)
 					return
@@ -163,7 +162,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T,
 ) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -487,7 +486,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code)
 		}
 		// read the response body.
-		bucketPolicyReadBuf, err := ioutil.ReadAll(recV4.Body)
+		bucketPolicyReadBuf, err := io.ReadAll(recV4.Body)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -525,7 +524,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code)
 		}
 		// read the response body.
-		bucketPolicyReadBuf, err = ioutil.ReadAll(recV2.Body)
+		bucketPolicyReadBuf, err = io.ReadAll(recV2.Body)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}

cmd/bucket-policy.go

@@ -27,6 +27,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
+	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -38,7 +39,8 @@ type PolicySys struct{}
 
 // Get returns stored bucket policy
 func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
-	return globalBucketMetadataSys.GetPolicyConfig(bucket)
+	policy, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
+	return policy, err
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
@@ -124,6 +126,20 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 
 	cloneHeader := r.Header.Clone()
 
+	if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
+		tag, _ := tags.ParseObjectTags(userTags)
+		if tag != nil {
+			tagMap := tag.ToMap()
+			keys := make([]string, 0, len(tagMap))
+			for k, v := range tagMap {
+				args[pathJoin("ExistingObjectTag", k)] = []string{v}
+				args[pathJoin("RequestObjectTag", k)] = []string{v}
+				keys = append(keys, k)
+			}
+			args["RequestObjectTagKeys"] = keys
+		}
+	}
+
 	for _, objLock := range []string{
 		xhttp.AmzObjectLockMode,
 		xhttp.AmzObjectLockLegalHold,
@@ -136,6 +152,9 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	}
 
 	for key, values := range cloneHeader {
+		if strings.EqualFold(key, xhttp.AmzObjectTagging) {
+			continue
+		}
 		if existingValues, found := args[key]; found {
 			args[key] = append(existingValues, values...)
 		} else {
@@ -168,6 +187,8 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	}
 
 	// JWT specific values
+	//
+	// Add all string claims
 	for k, v := range claims {
 		vStr, ok := v.(string)
 		if ok {
@@ -182,6 +203,21 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 			}
 		}
 	}
+	// Add groups claim which could be a list. This will ensure that the claim
+	// `jwt:groups` works.
+	if grpsVal, ok := claims["groups"]; ok {
+		if grpsIs, ok := grpsVal.([]interface{}); ok {
+			grps := []string{}
+			for _, gI := range grpsIs {
+				if g, ok := gI.(string); ok {
+					grps = append(grps, g)
+				}
+			}
+			if len(grps) > 0 {
+				args["groups"] = grps
+			}
+		}
+	}
 
 	return args
 }

cmd/bucket-replication-handlers.go

@@ -0,0 +1,410 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
+)
+
+// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
+// ----------
+// Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
+func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketReplicationConfig")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if globalIsGateway {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
+		return
+	}
+	replicationConfig, err := replication.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr.Description = err.Error()
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	if apiErr != noError {
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+	// Validate the received bucket replication config
+	if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	configData, err := xml.Marshal(replicationConfig)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketReplicationConfigHandler - GET Bucket replication configuration.
+// ----------
+// Gets the replication configuration for a bucket.
+func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketReplicationConfig")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseXML(w, configData)
+}
+
+// DeleteBucketReplicationConfigHandler - DELETE Bucket replication config.
+// ----------
+func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketReplicationConfig")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketReplicationMetricsHandler - GET Bucket replication metrics.
+// ----------
+// Gets the replication metrics for a bucket.
+func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketReplicationMetrics")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	var usageInfo BucketUsageInfo
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
+		usageInfo = dataUsageInfo.BucketsUsage[bucket]
+	}
+
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
+
+	enc := json.NewEncoder(w)
+	if err = enc.Encode(globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// ResetBucketReplicationStartHandler - starts a replication reset for all objects in a bucket which
+// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
+// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
+// remote target is entirely lost,and previously replicated objects need to be re-synced. If resync is
+// already in progress it returns an error
+func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ResetBucketReplicationStart")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	durationStr := r.URL.Query().Get("older-than")
+	arn := r.URL.Query().Get("arn")
+	resetID := r.URL.Query().Get("reset-id")
+	if resetID == "" {
+		resetID = mustGetUUID()
+	}
+	var (
+		days time.Duration
+		err  error
+	)
+	if durationStr != "" {
+		days, err = time.ParseDuration(durationStr)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{
+				Bucket: bucket,
+				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
+			}), r.URL)
+		}
+	}
+	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	if !config.HasExistingObjectReplication(arn) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL)
+		return
+	}
+
+	tgtArns := config.FilterTargetArns(
+		replication.ObjectOpts{
+			OpType:    replication.ResyncReplicationType,
+			TargetArn: arn,
+		})
+
+	if len(tgtArns) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("Remote target ARN %s missing or ineligible for replication resync", arn),
+		}), r.URL)
+		return
+	}
+
+	if len(tgtArns) > 1 && arn == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("ARN should be specified for replication reset"),
+		}), r.URL)
+		return
+	}
+	var rinfo ResyncTargetsInfo
+	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0])
+	target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
+	target.ResetID = resetID
+	rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
+		switch err.(type) {
+		case RemoteTargetConnectionErr:
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
+		default:
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		}
+	}
+	if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    err,
+		}), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(rinfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}
+
+// ResetBucketReplicationStatusHandler - returns the status of replication reset.
+// This API is a MinIO only extension
+func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ResetBucketReplicationStatus")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	arn := r.URL.Query().Get("arn")
+	var err error
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	globalReplicationPool.resyncState.RLock()
+	brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
+	globalReplicationPool.resyncState.RUnlock()
+	if !ok {
+		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
+		if err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+				Bucket: bucket,
+				Err:    fmt.Errorf("No replication resync status available for %s", arn),
+			}), r.URL)
+			return
+		}
+	}
+
+	var rinfo ResyncTargetsInfo
+	for tarn, st := range brs.TargetsMap {
+		if arn != "" && tarn != arn {
+			continue
+		}
+		rinfo.Targets = append(rinfo.Targets, ResyncTarget{
+			Arn:             tarn,
+			ResetID:         st.ResyncID,
+			StartTime:       st.StartTime,
+			EndTime:         st.EndTime,
+			ResyncStatus:    st.ResyncStatus.String(),
+			ReplicatedSize:  st.ReplicatedSize,
+			ReplicatedCount: st.ReplicatedCount,
+			FailedSize:      st.FailedSize,
+			FailedCount:     st.FailedCount,
+			Bucket:          st.Bucket,
+			Object:          st.Object,
+		})
+	}
+	data, err := json.Marshal(rinfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}

cmd/bucket-replication-stats.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"math"
 	"sync"
 	"time"
 
@@ -36,10 +37,12 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
 
 // ReplicationStats holds the global in-memory replication stats
 type ReplicationStats struct {
-	Cache      map[string]*BucketReplicationStats
-	UsageCache map[string]*BucketReplicationStats
-	sync.RWMutex
-	ulock sync.RWMutex
+	Cache           map[string]*BucketReplicationStats
+	UsageCache      map[string]*BucketReplicationStats
+	mostRecentStats BucketStatsMap
+	sync.RWMutex                 // mutex for Cache
+	ulock           sync.RWMutex // mutex for UsageCache
+	dlock           sync.RWMutex // mutex for mostRecentStats
 }
 
 // Delete deletes in-memory replication statistics for a bucket.
@@ -92,14 +95,23 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 		bs.Stats[arn] = b
 	}
 	switch status {
+	case replication.Pending:
+		if opType.IsDataReplication() && prevStatus != status {
+			b.PendingSize += n
+			b.PendingCount++
+		}
 	case replication.Completed:
 		switch prevStatus { // adjust counters based on previous state
+		case replication.Pending:
+			b.PendingCount--
 		case replication.Failed:
 			b.FailedCount--
 		}
-		if opType == replication.ObjectReplicationType {
+		if opType.IsDataReplication() {
 			b.ReplicatedSize += n
 			switch prevStatus {
+			case replication.Pending:
+				b.PendingSize -= n
 			case replication.Failed:
 				b.FailedSize -= n
 			}
@@ -108,10 +120,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 			}
 		}
 	case replication.Failed:
-		if opType == replication.ObjectReplicationType {
+		if opType.IsDataReplication() {
 			if prevStatus == replication.Pending {
 				b.FailedSize += n
 				b.FailedCount++
+				b.PendingSize -= n
+				b.PendingCount--
 			}
 		}
 	case replication.Replica:
@@ -135,6 +149,23 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	return st.Clone()
 }
 
+// GetAll returns replication metrics for all buckets at once.
+func (r *ReplicationStats) GetAll() map[string]BucketReplicationStats {
+	if r == nil {
+		return map[string]BucketReplicationStats{}
+	}
+
+	r.RLock()
+	defer r.RUnlock()
+
+	bucketReplicationStats := make(map[string]BucketReplicationStats, len(r.Cache))
+	for k, v := range r.Cache {
+		bucketReplicationStats[k] = v.Clone()
+	}
+
+	return bucketReplicationStats
+}
+
 // Get replication metrics for a bucket from this node since this node came up.
 func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	if r == nil {
@@ -159,9 +190,20 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 	}
 }
 
-// load replication metrics at cluster start from initial data usage
+// load replication metrics at cluster start from latest replication stats saved in .minio.sys/buckets/replication/node-name.stats
+// fallback to replication stats in data usage to be backward compatible
 func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
-	rTimer := time.NewTimer(time.Minute * 1)
+	m := make(map[string]*BucketReplicationStats)
+	if stats, err := globalReplicationPool.loadStatsFromDisk(); err == nil {
+		for b, st := range stats {
+			m[b] = &st
+		}
+		r.ulock.Lock()
+		r.UsageCache = m
+		r.ulock.Unlock()
+		return
+	}
+	rTimer := time.NewTimer(time.Second * 5)
 	defer rTimer.Stop()
 	var (
 		dui DataUsageInfo
@@ -174,17 +216,13 @@ outer:
 			return
 		case <-rTimer.C:
 			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
-			if err != nil {
-				continue
-			}
 			// If LastUpdate is set, data usage is available.
-			if !dui.LastUpdate.IsZero() {
+			if err == nil {
 				break outer
 			}
+			rTimer.Reset(time.Second * 5)
 		}
 	}
-
-	m := make(map[string]*BucketReplicationStats)
 	for bucket, usage := range dui.BucketsUsage {
 		b := &BucketReplicationStats{
 			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
@@ -203,6 +241,133 @@ outer:
 		}
 	}
 	r.ulock.Lock()
-	defer r.ulock.Unlock()
 	r.UsageCache = m
+	r.ulock.Unlock()
+}
+
+func (r *ReplicationStats) getAllCachedLatest() BucketStatsMap {
+	if r == nil {
+		return BucketStatsMap{}
+	}
+	r.dlock.RLock()
+	defer r.dlock.RUnlock()
+	return r.mostRecentStats
+}
+
+func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
+	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
+	bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
+
+	for bucket, u := range bucketsUsage {
+		bucketStats := make([]BucketStats, len(peerBucketStatsList))
+		for i, peerBucketStats := range peerBucketStatsList {
+			bucketStat, ok := peerBucketStats.Stats[bucket]
+			if !ok {
+				continue
+			}
+			bucketStats[i] = bucketStat
+		}
+		bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, u, bucketStats)
+	}
+	return bucketsReplicationStats
+}
+
+func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
+	if r == nil {
+		s = BucketReplicationStats{
+			Stats: make(map[string]*BucketReplicationStat),
+		}
+		return s
+	}
+
+	// accumulate cluster bucket stats
+	stats := make(map[string]*BucketReplicationStat)
+	var totReplicaSize int64
+	for _, bucketStat := range bucketStats {
+		totReplicaSize += bucketStat.ReplicationStats.ReplicaSize
+		for arn, stat := range bucketStat.ReplicationStats.Stats {
+			oldst := stats[arn]
+			if oldst == nil {
+				oldst = &BucketReplicationStat{}
+			}
+			stats[arn] = &BucketReplicationStat{
+				FailedCount:    stat.FailedCount + oldst.FailedCount,
+				FailedSize:     stat.FailedSize + oldst.FailedSize,
+				ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize,
+				Latency:        stat.Latency.merge(oldst.Latency),
+				PendingCount:   stat.PendingCount + oldst.PendingCount,
+				PendingSize:    stat.PendingSize + oldst.PendingSize,
+			}
+		}
+	}
+
+	// add initial usage stat to cluster stats
+	usageStat := globalReplicationStats.GetInitialUsage(bucket)
+
+	totReplicaSize += usageStat.ReplicaSize
+	for arn, stat := range usageStat.Stats {
+		st, ok := stats[arn]
+		if !ok {
+			st = &BucketReplicationStat{}
+			stats[arn] = st
+		}
+		st.ReplicatedSize += stat.ReplicatedSize
+		st.FailedSize += stat.FailedSize
+		st.FailedCount += stat.FailedCount
+		st.PendingSize += stat.PendingSize
+		st.PendingCount += stat.PendingCount
+	}
+
+	s = BucketReplicationStats{
+		Stats: make(map[string]*BucketReplicationStat, len(stats)),
+	}
+	var latestTotReplicatedSize int64
+	for _, st := range u.ReplicationInfo {
+		latestTotReplicatedSize += int64(st.ReplicatedSize)
+	}
+
+	// normalize computed real time stats with latest usage stat
+	for arn, tgtstat := range stats {
+		st := BucketReplicationStat{}
+		bu, ok := u.ReplicationInfo[arn]
+		if !ok {
+			bu = BucketTargetUsageInfo{}
+		}
+		// use in memory replication stats if it is ahead of usage info.
+		st.ReplicatedSize = int64(bu.ReplicatedSize)
+		if tgtstat.ReplicatedSize >= int64(bu.ReplicatedSize) {
+			st.ReplicatedSize = tgtstat.ReplicatedSize
+		}
+		s.ReplicatedSize += st.ReplicatedSize
+		// Reset FailedSize and FailedCount to 0 for negative overflows which can
+		// happen since data usage picture can lag behind actual usage state at the time of cluster start
+		st.FailedSize = int64(math.Max(float64(tgtstat.FailedSize), 0))
+		st.FailedCount = int64(math.Max(float64(tgtstat.FailedCount), 0))
+		st.PendingSize = int64(math.Max(float64(tgtstat.PendingSize), 0))
+		st.PendingCount = int64(math.Max(float64(tgtstat.PendingCount), 0))
+		st.Latency = tgtstat.Latency
+
+		s.Stats[arn] = &st
+		s.FailedSize += st.FailedSize
+		s.FailedCount += st.FailedCount
+		s.PendingCount += st.PendingCount
+		s.PendingSize += st.PendingSize
+	}
+	// normalize overall stats
+	s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
+	s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
+	r.dlock.Lock()
+	if len(r.mostRecentStats.Stats) == 0 {
+		r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
+	}
+	r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
+	r.mostRecentStats.Timestamp = UTCNow()
+	r.dlock.Unlock()
+	return s
+}
+
+// get the most current of in-memory replication stats  and data usage info from crawler.
+func (r *ReplicationStats) getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
+	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
+	return r.calculateBucketReplicationStats(bucket, u, bucketStats)
 }

cmd/bucket-replication-utils.go

@@ -21,12 +21,14 @@ import (
 	"bytes"
 	"fmt"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -503,28 +505,32 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 	}
 	var dsc ReplicateDecision
 	var tgtStatuses map[string]replication.StatusType
+	var purgeStatuses map[string]VersionPurgeStatusType
+
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
 			ObjectV: ObjectV{
 				ObjectName: oi.Name,
 				VersionID:  oi.VersionID,
 			},
-		}, oi, ObjectOptions{}, nil)
+		}, oi, ObjectOptions{VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)}, nil)
 	} else {
 		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
 			UserDefined: oi.UserDefined,
 		}, replication.HealReplicationType, ObjectOptions{}))
 	}
 	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
-
+	purgeStatuses = versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
 	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
-
+	tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
 	return ReplicateObjectInfo{
-		ObjectInfo:        oi,
-		OpType:            replication.HealReplicationType,
-		Dsc:               dsc,
-		ExistingObjResync: existingObjResync,
-		TargetStatuses:    tgtStatuses,
+		ObjectInfo:           oi,
+		OpType:               replication.HealReplicationType,
+		Dsc:                  dsc,
+		ExistingObjResync:    existingObjResync,
+		TargetStatuses:       tgtStatuses,
+		TargetPurgeStatuses:  purgeStatuses,
+		ReplicationTimestamp: tm,
 	}
 }
 
@@ -724,3 +730,51 @@ func parseSizeFromContentRange(h http.Header) (sz int64, err error) {
 	}
 	return int64(usz), nil
 }
+
+func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
+	opts.Verbose = q.Get("verbose") == "true"
+	opts.ARN = q.Get("arn")
+	opts.Prefix = q.Get("prefix")
+	return
+}
+
+const (
+	replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf"
+	mrfMetaFormat     = 1
+	mrfMetaVersionV1  = 1
+	mrfMetaVersion    = mrfMetaVersionV1
+)
+
+// MRFReplicateEntry mrf entry to save to disk
+type MRFReplicateEntry struct {
+	Bucket    string `json:"bucket" msg:"b"`
+	Object    string `json:"object" msg:"o"`
+	versionID string `json:"-"`
+}
+
+// MRFReplicateEntries has the map of MRF entries to save to disk
+type MRFReplicateEntries struct {
+	Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"`
+	Version int                          `json:"version" msg:"v"`
+}
+
+// ToMRFEntry returns the relevant info needed by MRF
+func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
+	return MRFReplicateEntry{
+		Bucket:    ri.Bucket,
+		Object:    ri.Name,
+		versionID: ri.VersionID,
+	}
+}
+
+func getReplicationStatsPath(nodeName string) string {
+	nodeStr := strings.ReplaceAll(nodeName, ":", "_")
+	return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + nodeStr + ".stats"
+}
+
+const (
+	replStatsMetaFormat   = 1
+	replStatsVersionV1    = 1
+	replStatsVersion      = replStatsVersionV1
+	replStatsSaveInterval = time.Minute * 5
+)

cmd/bucket-replication-utils_gen.go

@@ -260,6 +260,413 @@ func (z *BucketReplicationResyncStatus) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "e":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Entries")
+				return
+			}
+			if z.Entries == nil {
+				z.Entries = make(map[string]MRFReplicateEntry, zb0002)
+			} else if len(z.Entries) > 0 {
+				for key := range z.Entries {
+					delete(z.Entries, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 MRFReplicateEntry
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Entries")
+					return
+				}
+				var zb0003 uint32
+				zb0003, err = dc.ReadMapHeader()
+				if err != nil {
+					err = msgp.WrapError(err, "Entries", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						err = msgp.WrapError(err, "Entries", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "b":
+						za0002.Bucket, err = dc.ReadString()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+							return
+						}
+					case "o":
+						za0002.Object, err = dc.ReadString()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Object")
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001)
+							return
+						}
+					}
+				}
+				z.Entries[za0001] = za0002
+			}
+		case "v":
+			z.Version, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "e"
+	err = en.Append(0x82, 0xa1, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Entries)))
+	if err != nil {
+		err = msgp.WrapError(err, "Entries")
+		return
+	}
+	for za0001, za0002 := range z.Entries {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries")
+			return
+		}
+		// map header, size 2
+		// write "b"
+		err = en.Append(0x82, 0xa1, 0x62)
+		if err != nil {
+			return
+		}
+		err = en.WriteString(za0002.Bucket)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+			return
+		}
+		// write "o"
+		err = en.Append(0xa1, 0x6f)
+		if err != nil {
+			return
+		}
+		err = en.WriteString(za0002.Object)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "Object")
+			return
+		}
+	}
+	// write "v"
+	err = en.Append(0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Version)
+	if err != nil {
+		err = msgp.WrapError(err, "Version")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "e"
+	o = append(o, 0x82, 0xa1, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
+	for za0001, za0002 := range z.Entries {
+		o = msgp.AppendString(o, za0001)
+		// map header, size 2
+		// string "b"
+		o = append(o, 0x82, 0xa1, 0x62)
+		o = msgp.AppendString(o, za0002.Bucket)
+		// string "o"
+		o = append(o, 0xa1, 0x6f)
+		o = msgp.AppendString(o, za0002.Object)
+	}
+	// string "v"
+	o = append(o, 0xa1, 0x76)
+	o = msgp.AppendInt(o, z.Version)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "e":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Entries")
+				return
+			}
+			if z.Entries == nil {
+				z.Entries = make(map[string]MRFReplicateEntry, zb0002)
+			} else if len(z.Entries) > 0 {
+				for key := range z.Entries {
+					delete(z.Entries, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 MRFReplicateEntry
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Entries")
+					return
+				}
+				var zb0003 uint32
+				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Entries", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Entries", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "b":
+						za0002.Bucket, bts, err = msgp.ReadStringBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+							return
+						}
+					case "o":
+						za0002.Object, bts, err = msgp.ReadStringBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Object")
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001)
+							return
+						}
+					}
+				}
+				z.Entries[za0001] = za0002
+			}
+		case "v":
+			z.Version, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MRFReplicateEntries) Msgsize() (s int) {
+	s = 1 + 2 + msgp.MapHeaderSize
+	if z.Entries != nil {
+		for za0001, za0002 := range z.Entries {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
+		}
+	}
+	s += 2 + msgp.IntSize
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "b":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "o":
+			z.Object, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "b"
+	err = en.Append(0x82, 0xa1, 0x62)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "o"
+	err = en.Append(0xa1, 0x6f)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Object)
+	if err != nil {
+		err = msgp.WrapError(err, "Object")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "b"
+	o = append(o, 0x82, 0xa1, 0x62)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "o"
+	o = append(o, 0xa1, 0x6f)
+	o = msgp.AppendString(o, z.Object)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "b":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "o":
+			z.Object, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MRFReplicateEntry) Msgsize() (s int) {
+	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-replication-utils_gen_test.go

@@ -122,6 +122,232 @@ func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalMRFReplicateEntries(t *testing.T) {
+	v := MRFReplicateEntries{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeMRFReplicateEntries(t *testing.T) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeMRFReplicateEntries Msgsize() is inaccurate")
+	}
+
+	vn := MRFReplicateEntries{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalMRFReplicateEntry(t *testing.T) {
+	v := MRFReplicateEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeMRFReplicateEntry(t *testing.T) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeMRFReplicateEntry Msgsize() is inaccurate")
+	}
+
+	vn := MRFReplicateEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
 	v := ReplicateDecision{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-stats.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -26,7 +27,7 @@ import (
 // ReplicationLatency holds information of bucket operations latency, such us uploads
 type ReplicationLatency struct {
 	// Single & Multipart PUTs latency
-	UploadHistogram LastMinuteLatencies
+	UploadHistogram LastMinuteHistogram
 }
 
 // Merge two replication latency into a new one
@@ -41,7 +42,7 @@ func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) {
 	avg := rl.UploadHistogram.GetAvgData()
 	for k, v := range avg {
 		// Convert nanoseconds to milliseconds
-		ret[sizeTagToString(k)] = v.avg() / uint64(time.Millisecond)
+		ret[sizeTagToString(k)] = uint64(v.avg() / time.Millisecond)
 	}
 	return
 }
@@ -51,6 +52,12 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 	rl.UploadHistogram.Add(size, duration)
 }
 
+// BucketStatsMap captures bucket statistics for all buckets
+type BucketStatsMap struct {
+	Stats     map[string]BucketStats
+	Timestamp time.Time
+}
+
 // BucketStats bucket statistics
 type BucketStats struct {
 	ReplicationStats BucketReplicationStats
@@ -59,7 +66,7 @@ type BucketStats struct {
 // BucketReplicationStats represents inline replication statistics
 // such as pending, failed and completed bytes in total for a bucket
 type BucketReplicationStats struct {
-	Stats map[string]*BucketReplicationStat
+	Stats map[string]*BucketReplicationStat `json:",omitempty"`
 	// Pending size in bytes
 	PendingSize int64 `json:"pendingReplicationSize"`
 	// Completed size in bytes
@@ -123,3 +130,11 @@ func (bs *BucketReplicationStat) hasReplicationUsage() bool {
 		bs.PendingCount > 0 ||
 		bs.PendingSize > 0
 }
+
+func (brs BucketReplicationStats) String() string {
+	s := "ReplicatedSize=" + fmt.Sprintf("%d", brs.ReplicatedSize) + "+\n ReplicaSize=" + fmt.Sprintf("%d", brs.ReplicaSize)
+	for arn, st := range brs.Stats {
+		s += "\n arn: " + arn + " ReplicatedSize=" + fmt.Sprintf("%d", st.ReplicatedSize) + "  +::ReplicaSize=" + fmt.Sprintf("%d", st.ReplicaSize)
+	}
+	return s
+}

cmd/bucket-stats_gen.go

@@ -792,6 +792,264 @@ func (z *BucketStats) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *BucketStatsMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Stats")
+				return
+			}
+			if z.Stats == nil {
+				z.Stats = make(map[string]BucketStats, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 BucketStats
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				var zb0003 uint32
+				zb0003, err = dc.ReadMapHeader()
+				if err != nil {
+					err = msgp.WrapError(err, "Stats", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "ReplicationStats":
+						err = za0002.ReplicationStats.DecodeMsg(dc)
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001)
+							return
+						}
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
+		case "Timestamp":
+			z.Timestamp, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Timestamp")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Stats"
+	err = en.Append(0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Stats)))
+	if err != nil {
+		err = msgp.WrapError(err, "Stats")
+		return
+	}
+	for za0001, za0002 := range z.Stats {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Stats")
+			return
+		}
+		// map header, size 1
+		// write "ReplicationStats"
+		err = en.Append(0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
+		if err != nil {
+			return
+		}
+		err = za0002.ReplicationStats.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
+			return
+		}
+	}
+	// write "Timestamp"
+	err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Timestamp)
+	if err != nil {
+		err = msgp.WrapError(err, "Timestamp")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Stats"
+	o = append(o, 0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Stats)))
+	for za0001, za0002 := range z.Stats {
+		o = msgp.AppendString(o, za0001)
+		// map header, size 1
+		// string "ReplicationStats"
+		o = append(o, 0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
+		o, err = za0002.ReplicationStats.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
+			return
+		}
+	}
+	// string "Timestamp"
+	o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
+	o = msgp.AppendTime(o, z.Timestamp)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketStatsMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats")
+				return
+			}
+			if z.Stats == nil {
+				z.Stats = make(map[string]BucketStats, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 BucketStats
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				var zb0003 uint32
+				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Stats", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "ReplicationStats":
+						bts, err = za0002.ReplicationStats.UnmarshalMsg(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001)
+							return
+						}
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
+		case "Timestamp":
+			z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Timestamp")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BucketStatsMap) Msgsize() (s int) {
+	s = 1 + 6 + msgp.MapHeaderSize
+	if z.Stats != nil {
+		for za0001, za0002 := range z.Stats {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + 1 + 17 + za0002.ReplicationStats.Msgsize()
+		}
+	}
+	s += 10 + msgp.TimeSize
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *ReplicationLatency) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-stats_gen_test.go

@@ -348,6 +348,119 @@ func BenchmarkDecodeBucketStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalBucketStatsMap(t *testing.T) {
+	v := BucketStatsMap{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketStatsMap(t *testing.T) {
+	v := BucketStatsMap{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketStatsMap Msgsize() is inaccurate")
+	}
+
+	vn := BucketStatsMap{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketStatsMap(b *testing.B) {
+	v := BucketStatsMap{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicationLatency(t *testing.T) {
 	v := ReplicationLatency{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-targets.go

@@ -19,13 +19,14 @@ package cmd
 
 import (
 	"context"
-	"net/http"
+	"fmt"
+	"net/url"
 	"sync"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -36,7 +37,9 @@ import (
 )
 
 const (
-	defaultHealthCheckDuration = 30 * time.Second
+	defaultHealthCheckDuration = 5 * time.Second
+	// default interval for reload of all remote target endpoints
+	defaultHealthCheckReloadDuration = 30 * time.Minute
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -44,6 +47,113 @@ type BucketTargetSys struct {
 	sync.RWMutex
 	arnRemotesMap map[string]*TargetClient
 	targetsMap    map[string][]madmin.BucketTarget
+	hMutex        sync.RWMutex
+	hc            map[string]epHealth
+	hcClient      *madmin.AnonymousClient
+}
+
+// epHealth struct represents health of a replication target endpoint.
+type epHealth struct {
+	Endpoint string
+	Scheme   string
+	Online   bool
+}
+
+// isOffline returns current liveness result of remote target. Add endpoint to
+// healthcheck map if missing and default to online status
+func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
+	sys.hMutex.RLock()
+	defer sys.hMutex.RUnlock()
+	if h, ok := sys.hc[ep.Host]; ok {
+		return !h.Online
+	}
+	go sys.initHC(ep)
+	return false
+}
+
+func (sys *BucketTargetSys) initHC(ep *url.URL) {
+	sys.hMutex.Lock()
+	sys.hc[ep.Host] = epHealth{
+		Endpoint: ep.Host,
+		Scheme:   ep.Scheme,
+		Online:   true,
+	}
+	sys.hMutex.Unlock()
+}
+
+// newHCClient initializes an anonymous client for performing health check on the remote endpoints
+func newHCClient() *madmin.AnonymousClient {
+	clnt, e := madmin.NewAnonymousClientNoEndpoint()
+	if e != nil {
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
+		return nil
+	}
+	clnt.SetCustomTransport(globalRemoteTargetTransport)
+	return clnt
+}
+
+// heartBeat performs liveness check on remote endpoints.
+func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
+	hcTimer := time.NewTimer(defaultHealthCheckDuration)
+	defer hcTimer.Stop()
+	for {
+		select {
+		case <-hcTimer.C:
+			sys.hMutex.RLock()
+			var eps []madmin.ServerProperties
+			for _, ep := range sys.hc {
+				eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
+			}
+			sys.hMutex.RUnlock()
+
+			if len(eps) > 0 {
+				cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+				m := map[string]epHealth{}
+				for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
+					var online bool
+					if result.Error == nil {
+						online = result.Online
+					}
+					m[result.Endpoint.Host] = epHealth{
+						Endpoint: result.Endpoint.Host,
+						Scheme:   result.Endpoint.Scheme,
+						Online:   online,
+					}
+				}
+				cancel()
+				sys.hMutex.Lock()
+				sys.hc = m
+				sys.hMutex.Unlock()
+			}
+			hcTimer.Reset(defaultHealthCheckDuration)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// periodically rebuild the healthcheck map from list of targets to clear
+// out stale endpoints
+func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
+	m := make(map[string]epHealth)
+	tgts := sys.ListTargets(ctx, "", "")
+	for _, t := range tgts {
+		if _, ok := m[t.Endpoint]; !ok {
+			scheme := "http"
+			if t.Secure {
+				scheme = "https"
+			}
+			m[t.Endpoint] = epHealth{
+				Online:   true,
+				Endpoint: t.Endpoint,
+				Scheme:   scheme,
+			}
+		}
+	}
+	sys.hMutex.Lock()
+	// swap out the map
+	sys.hc = m
+	sys.hMutex.Unlock()
 }
 
 // ListTargets lists bucket targets across tenant or for individual bucket, and returns
@@ -92,9 +202,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
 		return
 	}
 	for _, t := range tgts {
-		if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-			tgt.healthCancelFn()
-		}
 		delete(sys.arnRemotesMap, t.Arn)
 	}
 	delete(sys.targetsMap, bucket)
@@ -117,18 +224,15 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
 			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 		}
-		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 	}
 	if tgt.Type == madmin.ReplicationService {
-		if !globalIsErasure {
-			return NotImplemented{Message: "Replication is not implemented in " + getMinioMode()}
-		}
 		if !globalBucketVersioningSys.Enabled(bucket) {
 			return BucketReplicationSourceNotVersioned{Bucket: bucket}
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
@@ -156,10 +260,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
-	// cancel health check for previous target client to avoid leak.
-	if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
-		prevClnt.healthCancelFn()
-	}
+
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
 	sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
@@ -184,9 +285,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	if globalIsGateway {
 		return nil
 	}
-	if !globalIsErasure {
-		return NotImplemented{Message: "Replication is not implemented in " + getMinioMode()}
-	}
 
 	if arnStr == "" {
 		return BucketRemoteArnInvalid{Bucket: bucket}
@@ -201,7 +299,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
 		if err == nil {
-			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{}) {
+			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{OpType: replication.AllReplicationType}) {
 				if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) {
 					sys.RLock()
 					_, ok := sys.arnRemotesMap[arnStr]
@@ -234,9 +332,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	sys.targetsMap[bucket] = targets
-	if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
-		tgt.healthCancelFn()
-	}
 	delete(sys.arnRemotesMap, arnStr)
 	sys.updateBandwidthLimit(bucket, 0)
 	return nil
@@ -265,11 +360,29 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
 }
 
 // NewBucketTargetSys - creates new replication system.
-func NewBucketTargetSys() *BucketTargetSys {
-	return &BucketTargetSys{
+func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
+	sys := &BucketTargetSys{
 		arnRemotesMap: make(map[string]*TargetClient),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
+		hc:            make(map[string]epHealth),
+		hcClient:      newHCClient(),
 	}
+	// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
+	go func() {
+		rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
+		defer rTimer.Stop()
+		for {
+			select {
+			case <-rTimer.C:
+				sys.reloadHealthCheckers(ctx)
+				rTimer.Reset(defaultHealthCheckReloadDuration)
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	go sys.heartBeat(ctx)
+	return sys
 }
 
 // UpdateAllTargets updates target to reflect metadata updates
@@ -279,17 +392,17 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if tgts == nil || tgts.Empty() {
-		// remove target and arn association
-		if tgts, ok := sys.targetsMap[bucket]; ok {
-			for _, t := range tgts {
-				if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-					tgt.healthCancelFn()
-				}
-				delete(sys.arnRemotesMap, t.Arn)
-			}
+
+	// Remove existingtarget and arn association
+	if tgts, ok := sys.targetsMap[bucket]; ok {
+		for _, t := range tgts {
+			delete(sys.arnRemotesMap, t.Arn)
 		}
 		delete(sys.targetsMap, bucket)
+	}
+
+	// No need for more if not adding anything
+	if tgts == nil || tgts.Empty() {
 		sys.updateBandwidthLimit(bucket, 0)
 		return
 	}
@@ -331,25 +444,16 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 	sys.targetsMap[bucket.Name] = cfg.Targets
 }
 
-// getRemoteTargetInstanceTransport contains a singleton roundtripper.
-var (
-	getRemoteTargetInstanceTransport     http.RoundTripper
-	getRemoteTargetInstanceTransportOnce sync.Once
-)
-
 // Returns a minio-go Client configured to access remote host described in replication target config.
 func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
 	config := tcfg.Credentials
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
-	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = NewRemoteTargetHTTPTransport()
-	})
 
 	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
 		Region:    tcfg.Region,
-		Transport: getRemoteTargetInstanceTransport,
+		Transport: globalRemoteTargetTransport,
 	})
 	if err != nil {
 		return nil, err
@@ -358,10 +462,6 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
 	}
-	cancelFn, err := api.HealthCheck(hcDuration)
-	if err != nil {
-		return nil, err
-	}
 	tc := &TargetClient{
 		Client:              api,
 		healthCheckDuration: hcDuration,
@@ -369,9 +469,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 		Bucket:              tcfg.TargetBucket,
 		StorageClass:        tcfg.StorageClass,
 		disableProxy:        tcfg.DisableProxy,
-		healthCancelFn:      cancelFn,
 		ARN:                 tcfg.Arn,
 		ResetID:             tcfg.ResetID,
+		Endpoint:            tcfg.Endpoint,
+		Secure:              tcfg.Secure,
 	}
 	return tc, nil
 }
@@ -445,7 +546,8 @@ type TargetClient struct {
 	replicateSync       bool
 	StorageClass        string // storage class on remote
 	disableProxy        bool
-	healthCancelFn      context.CancelFunc // cancellation function for client healthcheck
-	ARN                 string             // ARN to uniquely identify remote target
+	ARN                 string // ARN to uniquely identify remote target
 	ResetID             string
+	Endpoint            string
+	Secure              bool
 }

cmd/bucket-versioning-handler.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/xml"
 	"io"
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
@@ -63,28 +65,28 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if globalSiteReplicationSys.isEnabled() {
+	if globalSiteReplicationSys.isEnabled() && !v.Enabled() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
-			Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
-			HTTPStatusCode: http.StatusConflict,
+			Description:    "Cluster replication is enabled on this site, versioning cannot be suspended on bucket.",
+			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL)
 		return
 	}
 
-	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
+	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && (v.Suspended() || v.PrefixesExcluded()) {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
-			Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
-			HTTPStatusCode: http.StatusConflict,
+			Description:    "An Object Lock configuration is present on this bucket, versioning cannot be suspended.",
+			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL)
 		return
 	}
 	if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
-			Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
-			HTTPStatusCode: http.StatusConflict,
+			Description:    "A replication configuration is present on this bucket, bucket wide versioning cannot be suspended.",
+			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL)
 		return
 	}
@@ -95,7 +97,23 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:       madmin.SRBucketMetaTypeVersionConfig,
+		Bucket:     bucket,
+		Versioning: &cfgStr,
+		UpdatedAt:  updatedAt,
+	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -125,7 +143,7 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-versioning.go

@@ -17,44 +17,69 @@
 
 package cmd
 
-import "github.com/minio/minio/internal/bucket/versioning"
+import (
+	"strings"
+
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/logger"
+)
 
 // BucketVersioningSys - policy subsystem.
 type BucketVersioningSys struct{}
 
 // Enabled enabled versioning?
 func (sys *BucketVersioningSys) Enabled(bucket string) bool {
-	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	vc, err := sys.Get(bucket)
 	if err != nil {
-		return false
+		logger.CriticalIf(GlobalContext, err)
 	}
 	return vc.Enabled()
 }
 
+// PrefixEnabled returns true is versioning is enabled at bucket level and if
+// the given prefix doesn't match any excluded prefixes pattern. This is
+// part of a MinIO versioning configuration extension.
+func (sys *BucketVersioningSys) PrefixEnabled(bucket, prefix string) bool {
+	vc, err := sys.Get(bucket)
+	if err != nil {
+		logger.CriticalIf(GlobalContext, err)
+	}
+	return vc.PrefixEnabled(prefix)
+}
+
 // Suspended suspended versioning?
 func (sys *BucketVersioningSys) Suspended(bucket string) bool {
-	vc, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	vc, err := sys.Get(bucket)
 	if err != nil {
-		return false
+		logger.CriticalIf(GlobalContext, err)
 	}
 	return vc.Suspended()
 }
 
+// PrefixSuspended returns true if the given prefix matches an excluded prefix
+// pattern. This is part of a MinIO versioning configuration extension.
+func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
+	vc, err := sys.Get(bucket)
+	if err != nil {
+		logger.CriticalIf(GlobalContext, err)
+	}
+
+	return vc.PrefixSuspended(prefix)
+}
+
 // Get returns stored bucket policy
 func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-		return nil, NotImplemented{}
+		// Gateway does not implement versioning.
+		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
 	}
-	return globalBucketMetadataSys.GetVersioningConfig(bucket)
-}
 
-// Reset BucketVersioningSys to initial state.
-func (sys *BucketVersioningSys) Reset() {
-	// There is currently no internal state.
+	if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) {
+		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
+	}
+
+	vcfg, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+	return vcfg, err
 }
 
 // NewBucketVersioningSys - creates new versioning system.

cmd/build-constants.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -37,4 +37,7 @@ var (
 
 	// ShortCommitID - first 12 characters from CommitID.
 	ShortCommitID = "DEVELOPMENT.GOGET"
+
+	// CopyrightYear - dynamic value of the copyright end year
+	CopyrightYear = "0000"
 )

cmd/callhome.go

@@ -0,0 +1,150 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/logger"
+	uatomic "go.uber.org/atomic"
+)
+
+const (
+	// callhomeSchemaVersion1 is callhome schema version 1
+	callhomeSchemaVersion1 = "1"
+
+	// callhomeSchemaVersion is current callhome schema version.
+	callhomeSchemaVersion = callhomeSchemaVersion1
+
+	// callhomeCycleDefault is the default interval between two callhome cycles (24hrs)
+	callhomeCycleDefault = 24 * time.Hour
+)
+
+// CallhomeInfo - Contains callhome information
+type CallhomeInfo struct {
+	SchemaVersion string             `json:"schema_version"`
+	AdminInfo     madmin.InfoMessage `json:"admin_info"`
+}
+
+var (
+	enableCallhome            = uatomic.NewBool(false)
+	callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	callhomeFreq              = uatomic.NewDuration(callhomeCycleDefault)
+)
+
+func updateCallhomeParams(ctx context.Context, objAPI ObjectLayer) {
+	alreadyEnabled := enableCallhome.Load()
+	enableCallhome.Store(globalCallhomeConfig.Enable)
+	callhomeFreq.Store(globalCallhomeConfig.Frequency)
+
+	// If callhome was disabled earlier and has now been enabled,
+	// initialize the callhome process again.
+	if !alreadyEnabled && enableCallhome.Load() {
+		initCallhome(ctx, objAPI)
+	}
+}
+
+// initCallhome will start the callhome task in the background.
+func initCallhome(ctx context.Context, objAPI ObjectLayer) {
+	go func() {
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		// Leader node (that successfully acquires the lock inside runCallhome)
+		// will keep performing the callhome. If the leader goes down for some reason,
+		// the lock will be released and another node will acquire it and take over
+		// because of this loop.
+		for {
+			runCallhome(ctx, objAPI)
+			if !enableCallhome.Load() {
+				return
+			}
+
+			// callhome running on a different node.
+			// sleep for some time and try again.
+			duration := time.Duration(r.Float64() * float64(callhomeFreq.Load()))
+			if duration < time.Second {
+				// Make sure to sleep atleast a second to avoid high CPU ticks.
+				duration = time.Second
+			}
+			time.Sleep(duration)
+
+			if !enableCallhome.Load() {
+				return
+			}
+		}
+	}()
+}
+
+func runCallhome(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 callhome is running on the cluster.
+	locker := objAPI.NewNSLock(minioMetaBucket, "callhome/runCallhome.lock")
+	lkctx, err := locker.GetLock(ctx, callhomeLeaderLockTimeout)
+	if err != nil {
+		return
+	}
+
+	ctx = lkctx.Context()
+	defer locker.Unlock(lkctx.Cancel)
+
+	callhomeTimer := time.NewTimer(callhomeFreq.Load())
+	defer callhomeTimer.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-callhomeTimer.C:
+			if !enableCallhome.Load() {
+				// Stop the processing as callhome got disabled
+				return
+			}
+			performCallhome(ctx)
+
+			// Reset the timer for next cycle.
+			callhomeTimer.Reset(callhomeFreq.Load())
+		}
+	}
+}
+
+func performCallhome(ctx context.Context) {
+	err := sendCallhomeInfo(
+		CallhomeInfo{
+			SchemaVersion: callhomeSchemaVersion,
+			AdminInfo:     getServerInfo(ctx, nil),
+		})
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
+	}
+}
+
+const (
+	callhomeURL    = "https://subnet.min.io/api/callhome"
+	callhomeURLDev = "http://localhost:9000/api/callhome"
+)
+
+func sendCallhomeInfo(ch CallhomeInfo) error {
+	url := callhomeURL
+	if globalIsCICD {
+		url = callhomeURLDev
+	}
+	_, err := globalSubnetConfig.Post(url, ch)
+	return err
+}

cmd/common-main.go

@@ -27,7 +27,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -48,6 +47,7 @@ import (
 	"github.com/inconshreveable/mousetrap"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"
 	consoleCerts "github.com/minio/console/pkg/certs"
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
@@ -102,6 +102,11 @@ func init() {
 		PersistOnFailure: false,
 	}
 
+	t, _ := minioVersionToReleaseTime(Version)
+	if !t.IsZero() {
+		globalVersionUnix = uint64(t.Unix())
+	}
+
 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
 
 	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()
@@ -190,34 +195,16 @@ func minioConfigToConsoleFeatures() {
 		os.Setenv("CONSOLE_PROMETHEUS_URL", value)
 		if value := env.Get("MINIO_PROMETHEUS_JOB_ID", "minio-job"); value != "" {
 			os.Setenv("CONSOLE_PROMETHEUS_JOB_ID", value)
+			// Support additional labels for more granular filtering.
+			if value := env.Get("MINIO_PROMETHEUS_EXTRA_LABELS", ""); value != "" {
+				os.Setenv("CONSOLE_PROMETHEUS_EXTRA_LABELS", value)
+			}
 		}
 	}
 	// Enable if LDAP is enabled.
 	if globalLDAPConfig.Enabled {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
-	// if IDP is enabled, set IDP environment variables
-	if globalOpenIDConfig.ProviderCfgs[config.Default] != nil {
-		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.ProviderCfgs[config.Default].URL.String())
-		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ProviderCfgs[config.Default].ClientID)
-		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ProviderCfgs[config.Default].ClientSecret)
-		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
-		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ProviderCfgs[config.Default].ClientID)
-		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.ProviderCfgs[config.Default].DiscoveryDoc.ScopesSupported, ","))
-		if globalOpenIDConfig.ProviderCfgs[config.Default].ClaimUserinfo {
-			os.Setenv("CONSOLE_IDP_USERINFO", config.EnableOn)
-		}
-		if globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURIDynamic {
-			// Enable dynamic redirect-uri's based on incoming 'host' header,
-			// Overrides any other callback URL.
-			os.Setenv("CONSOLE_IDP_CALLBACK_DYNAMIC", config.EnableOn)
-		}
-		if globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURI != "" {
-			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURI)
-		} else {
-			os.Setenv("CONSOLE_IDP_CALLBACK", getConsoleEndpoints()[0]+"/oauth_callback")
-		}
-	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
 	if globalSubnetConfig.License != "" {
@@ -226,11 +213,34 @@ func minioConfigToConsoleFeatures() {
 	if globalSubnetConfig.APIKey != "" {
 		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
 	}
-	if globalSubnetConfig.Proxy != "" {
-		os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.Proxy)
+	if globalSubnetConfig.ProxyURL != nil {
+		os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.ProxyURL.String())
 	}
 }
 
+func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
+	m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
+	for name, cfg := range globalOpenIDConfig.ProviderCfgs {
+		callback := getConsoleEndpoints()[0] + "/oauth_callback"
+		if cfg.RedirectURI != "" {
+			callback = cfg.RedirectURI
+		}
+		m[name] = consoleoauth2.ProviderConfig{
+			URL:                     cfg.URL.String(),
+			DisplayName:             cfg.DisplayName,
+			ClientID:                cfg.ClientID,
+			ClientSecret:            cfg.ClientSecret,
+			HMACSalt:                globalDeploymentID,
+			HMACPassphrase:          cfg.ClientID,
+			Scopes:                  strings.Join(cfg.DiscoveryDoc.ScopesSupported, ","),
+			Userinfo:                cfg.ClaimUserinfo,
+			RedirectCallbackDynamic: cfg.RedirectURIDynamic,
+			RedirectCallback:        callback,
+		}
+	}
+	return m
+}
+
 func initConsoleServer() (*restapi.Server, error) {
 	// unset all console_ environment variables.
 	for _, cenv := range env.List(consolePrefix) {
@@ -264,6 +274,12 @@ func initConsoleServer() (*restapi.Server, error) {
 		api.Logger = noLog
 	}
 
+	// Pass in console application config. This needs to happen before the
+	// ConfigureAPI() call.
+	restapi.GlobalMinIOConfig = restapi.MinIOConfig{
+		OpenIDProviders: buildOpenIDConsoleConfig(),
+	}
+
 	server := restapi.NewServer(api)
 	// register all APIs
 	server.ConfigureAPI()
@@ -421,7 +437,6 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		if err != nil {
 			logger.FatalIf(err, "Unable to get free port for console on the host")
 		}
-		globalMinioConsolePortAuto = true
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
 
@@ -486,8 +501,13 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 			Skip: true,
 		}, nil
 	}
-	const envSeparator = "="
-	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	if strings.HasPrefix(envEntry, "#") {
+		// Skip commented lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
 	if len(envTokens) != 2 {
 		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
 	}
@@ -495,13 +515,6 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 	key := envTokens[0]
 	val := envTokens[1]
 
-	if strings.HasPrefix(key, "#") {
-		// Skip commented lines
-		return envKV{
-			Skip: true,
-		}, nil
-	}
-
 	// Remove quotes from the value if found
 	if len(val) >= 2 {
 		quote := val[0]
@@ -520,11 +533,8 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 // the environment values from a file, in the form "key, value".
 // in a structured form.
 func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
-	f, err := os.Open(envConfigFile)
+	f, err := Open(envConfigFile)
 	if err != nil {
-		if os.IsNotExist(err) { // ignore if file doesn't exist.
-			return nil, nil
-		}
 		return nil, err
 	}
 	defer f.Close()
@@ -553,7 +563,7 @@ func readFromSecret(sp string) (string, error) {
 	if isFile(pathJoin("/run/secrets/", sp)) {
 		sp = pathJoin("/run/secrets/", sp)
 	}
-	credBuf, err := ioutil.ReadFile(sp)
+	credBuf, err := os.ReadFile(sp)
 	if err != nil {
 		if os.IsNotExist(err) { // ignore if file doesn't exist.
 			return "", nil
@@ -620,7 +630,7 @@ func loadEnvVarsFromFiles() {
 
 	if env.IsSet(config.EnvConfigEnvFile) {
 		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
-		if err != nil {
+		if err != nil && !os.IsNotExist(err) {
 			logger.Fatal(err, "Unable to read the config environment file")
 		}
 		for _, ekv := range ekvs {
@@ -783,17 +793,22 @@ func handleCommonEnvVars() {
 		}
 		globalActiveCred = cred
 	}
+}
 
+// Initialize KMS global variable after valiadating and loading the configuration.
+// It depends on KMS env variables and global cli flags.
+func handleKMSConfig() {
 	switch {
 	case env.IsSet(config.EnvKMSSecretKey) && env.IsSet(config.EnvKESEndpoint):
 		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSSecretKey, config.EnvKESEndpoint))
 	}
 
 	if env.IsSet(config.EnvKMSSecretKey) {
-		GlobalKMS, err = kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
+		KMS, err := kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
 		}
+		GlobalKMS = KMS
 	}
 	if env.IsSet(config.EnvKESEndpoint) {
 		var endpoints []string
@@ -813,43 +828,56 @@ func handleCommonEnvVars() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		// Manually load the certificate and private key into memory.
-		// We need to check whether the private key is encrypted, and
-		// if so, decrypt it using the user-provided password.
-		certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
-		}
-		keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
-		}
-		privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
-		if len(rest) != 0 {
-			logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
-		}
-		if x509.IsEncryptedPEMBlock(privateKeyPEM) {
-			keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
-			if err != nil {
-				logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
-			}
-			keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
-		}
-		certificate, err := tls.X509KeyPair(certBytes, keyBytes)
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
-		}
 		rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
 		if err != nil {
 			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
+		loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
+			// Manually load the certificate and private key into memory.
+			// We need to check whether the private key is encrypted, and
+			// if so, decrypt it using the user-provided password.
+			certBytes, err := os.ReadFile(certFile)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+			}
+			keyBytes, err := os.ReadFile(keyFile)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+			}
+			privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+			if len(rest) != 0 {
+				return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
+			}
+			if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+				keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+				}
+				keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+			}
+			certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+			}
+			return certificate, nil
+		}
+
+		reloadCertEvents := make(chan tls.Certificate, 1)
+		certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
+		if err != nil {
+			logger.Fatal(err, "Failed to load KES client certificate")
+		}
+		certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
+		certificate.Notify(reloadCertEvents)
+
 		defaultKeyID := env.Get(config.EnvKESKeyName, "")
 		KMS, err := kms.NewWithConfig(kms.Config{
-			Endpoints:    endpoints,
-			DefaultKeyID: defaultKeyID,
-			Certificate:  certificate,
-			RootCAs:      rootCAs,
+			Endpoints:        endpoints,
+			DefaultKeyID:     defaultKeyID,
+			Certificate:      certificate,
+			ReloadCertEvents: reloadCertEvents,
+			RootCAs:          rootCAs,
 		})
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
@@ -859,7 +887,7 @@ func handleCommonEnvVars() {
 		// This implicitly checks that we can communicate to KES. We don't treat
 		// a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+		if err = KMS.CreateKey(context.Background(), defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
 		GlobalKMS = KMS
@@ -899,7 +927,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	// Therefore, we read all filenames in the cert directory and check
 	// for each directory whether it contains a public.crt and private.key.
 	// If so, we try to add it to certificate manager.
-	root, err := os.Open(globalCertsDir.Get())
+	root, err := Open(globalCertsDir.Get())
 	if err != nil {
 		return nil, nil, false, err
 	}
@@ -918,7 +946,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 			continue
 		}
 		if file.Mode()&os.ModeSymlink == os.ModeSymlink {
-			file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
+			file, err = Stat(filepath.Join(root.Name(), file.Name()))
 			if err != nil {
 				// not accessible ignore
 				continue

cmd/common-main_test.go

@@ -19,7 +19,7 @@ package cmd
 
 import (
 	"errors"
-	"io/ioutil"
+	"os"
 	"reflect"
 	"testing"
 )
@@ -45,7 +45,7 @@ func Test_readFromSecret(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			tmpfile, err := ioutil.TempFile("", "testfile")
+			tmpfile, err := os.CreateTemp("", "testfile")
 			if err != nil {
 				t.Error(err)
 			}
@@ -136,6 +136,7 @@ export MINIO_ROOT_PASSWORD=minio123`,
 		},
 		{
 			`
+# simple comment
 # MINIO_ROOT_USER=minioadmin
 # MINIO_ROOT_PASSWORD=minioadmin
 MINIO_ROOT_USER=minio
@@ -156,7 +157,7 @@ MINIO_ROOT_PASSWORD=minio123`,
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			tmpfile, err := ioutil.TempFile("", "testfile")
+			tmpfile, err := os.CreateTemp("", "testfile")
 			if err != nil {
 				t.Error(err)
 			}

cmd/config-common.go

@@ -21,7 +21,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/minio/minio/internal/hash"
@@ -41,7 +41,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
 	}
 	defer r.Close()
 
-	buf, err := ioutil.ReadAll(r)
+	buf, err := io.ReadAll(r)
 	if err != nil {
 		return nil, ObjectInfo{}, err
 	}

cmd/config-current.go

@@ -25,18 +25,22 @@ import (
 	"sync"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
 	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/config/etcd"
 	"github.com/minio/minio/internal/config/heal"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/scanner"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/config/subnet"
@@ -55,7 +59,9 @@ func initHelp() {
 		config.IdentityLDAPSubSys:   xldap.DefaultKVS,
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
 		config.IdentityTLSSubSys:    xtls.DefaultKVS,
+		config.IdentityPluginSubSys: idplugin.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
+		config.PolicyPluginSubSys:   polplugin.DefaultKVS,
 		config.SiteSubSys:           config.DefaultSiteKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
@@ -63,15 +69,16 @@ func initHelp() {
 		config.LoggerWebhookSubSys:  logger.DefaultLoggerWebhookKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
-		config.HealSubSys:           heal.DefaultKVS,
 		config.ScannerSubSys:        scanner.DefaultKVS,
 		config.SubnetSubSys:         subnet.DefaultKVS,
+		config.CallhomeSubSys:       callhome.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
 	}
 	if globalIsErasure {
 		kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
+		kvs[config.HealSubSys] = heal.DefaultKVS
 	}
 	config.RegisterDefaultKVS(kvs)
 
@@ -107,17 +114,17 @@ func initHelp() {
 			Description: "enable X.509 TLS certificate SSO support",
 		},
 		config.HelpKV{
-			Key:         config.PolicyOPASubSys,
-			Description: "[DEPRECATED] enable external OPA for policy enforcement",
+			Key:         config.IdentityPluginSubSys,
+			Description: "enable Identity Plugin via external hook",
+		},
+		config.HelpKV{
+			Key:         config.PolicyPluginSubSys,
+			Description: "enable Access Management Plugin for policy enforcement",
 		},
 		config.HelpKV{
 			Key:         config.APISubSys,
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
-		config.HelpKV{
-			Key:         config.HealSubSys,
-			Description: "manage object healing frequency and bitrot verification checks",
-		},
 		config.HelpKV{
 			Key:         config.ScannerSubSys,
 			Description: "manage namespace scanning for usage calculation, lifecycle, healing and more",
@@ -193,15 +200,22 @@ func initHelp() {
 			Description: "set subnet config for the cluster e.g. api key",
 			Optional:    true,
 		},
+		config.HelpKV{
+			Key:         config.CallhomeSubSys,
+			Type:        "string",
+			Description: "enable callhome for the cluster",
+			Optional:    true,
+		},
 	}
 
 	if globalIsErasure {
-		helpSubSys = append(helpSubSys, config.HelpKV{})
-		copy(helpSubSys[2:], helpSubSys[1:])
-		helpSubSys[1] = config.HelpKV{
+		helpSubSys = append(helpSubSys, config.HelpKV{
 			Key:         config.StorageClassSubSys,
 			Description: "define object level redundancy",
-		}
+		}, config.HelpKV{
+			Key:         config.HealSubSys,
+			Description: "manage object healing frequency and bitrot verification checks",
+		})
 	}
 
 	helpMap := map[string]config.HelpKVS{
@@ -218,7 +232,9 @@ func initHelp() {
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.IdentityTLSSubSys:    xtls.Help,
+		config.IdentityPluginSubSys: idplugin.Help,
 		config.PolicyOPASubSys:      opa.Help,
+		config.PolicyPluginSubSys:   polplugin.Help,
 		config.LoggerWebhookSubSys:  logger.Help,
 		config.AuditWebhookSubSys:   logger.HelpWebhook,
 		config.AuditKafkaSubSys:     logger.HelpKafka,
@@ -233,6 +249,7 @@ func initHelp() {
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
 		config.SubnetSubSys:         subnet.HelpSubnet,
+		config.CallhomeSubSys:       callhome.HelpCallhome,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
@@ -243,6 +260,10 @@ func initHelp() {
 			Key:         config.RegionSubSys,
 			Description: "[DEPRECATED - use `site` instead] label the location of the server",
 		},
+		config.PolicyOPASubSys: {
+			Key:         config.PolicyOPASubSys,
+			Description: "[DEPRECATED - use `policy_plugin` instead] enable external OPA for policy enforcement",
+		},
 	}
 
 	config.RegisterHelpDeprecatedSubSys(deprecatedHelpKVMap)
@@ -256,10 +277,6 @@ var (
 
 func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) error {
 	switch subSys {
-	case config.CredentialsSubSys:
-		if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
-			return err
-		}
 	case config.SiteSubSys:
 		if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
 			return err
@@ -269,14 +286,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	case config.StorageClassSubSys:
-		if globalIsErasure {
-			if objAPI == nil {
-				return errServerNotInitialized
-			}
-			for _, setDriveCount := range objAPI.SetDriveCounts() {
-				if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
-					return err
-				}
+		if objAPI == nil {
+			return errServerNotInitialized
+		}
+		for _, setDriveCount := range objAPI.SetDriveCounts() {
+			if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
+				return err
 			}
 		}
 	case config.CacheSubSys:
@@ -315,7 +330,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			etcdClnt.Close()
 		}
 	case config.IdentityOpenIDSubSys:
-		if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys],
+		if _, err := openid.LookupConfig(s,
 			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 			return err
 		}
@@ -335,14 +350,34 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 		if _, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default]); err != nil {
 			return err
 		}
+	case config.IdentityPluginSubSys:
+		if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
+			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
+			return err
+		}
 	case config.SubnetSubSys:
-		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default]); err != nil {
+		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], nil); err != nil {
+			return err
+		}
+	case config.CallhomeSubSys:
+		if _, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]); err != nil {
 			return err
 		}
 	case config.PolicyOPASubSys:
-		if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+		// In case legacy OPA config is being set, we treat it as if the
+		// AuthZPlugin is being set.
+		subSys = config.PolicyPluginSubSys
+		fallthrough
+	case config.PolicyPluginSubSys:
+		if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
 			NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
 			return err
+		} else if ppargs.URL == nil {
+			// Check if legacy opa is configured.
+			if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+				NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+				return err
+			}
 		}
 	default:
 		if config.LoggerSubSystems.Contains(subSys) {
@@ -353,7 +388,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	}
 
 	if config.NotifySubSystems.Contains(subSys) {
-		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs(), subSys); err != nil {
+		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewGatewayHTTPTransport()); err != nil {
 			return err
 		}
 	}
@@ -428,14 +463,12 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
-		if globalEtcdClient == nil {
-			globalEtcdClient, err = etcd.New(etcdCfg)
-			if err != nil {
-				if globalIsGateway {
-					logger.FatalIf(err, "Unable to initialize etcd config")
-				} else {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
-				}
+		globalEtcdClient, err = etcd.New(etcdCfg)
+		if err != nil {
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize etcd config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 			}
 		}
 
@@ -443,7 +476,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			if globalDNSConfig != nil {
 				// if global DNS is already configured, indicate with a warning, incase
 				// users are confused.
-				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
+				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
 			} else {
 				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 					dns.DomainNames(globalDomainNames),
@@ -475,16 +508,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
-	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
-	}
-
-	// Initialize remote instance transport once.
-	getRemoteInstanceTransportOnce.Do(func() {
-		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
-	})
-
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
 		if globalIsGateway {
@@ -517,36 +540,14 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
-	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys],
-		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
-	}
-
-	opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OPA: %w", err))
-	}
-	globalPolicyOPA = opa.New(opaCfg)
-
-	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
-		globalRootCAs)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
-	}
-
-	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
+	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
-	}
+	transport := NewGatewayHTTPTransport()
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
+	globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -562,27 +563,32 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 }
 
 func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s config.Config, subSys string) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	setDriveCounts := objAPI.SetDriveCounts()
 	switch subSys {
 	case config.APISubSys:
 		apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 		}
-		var setDriveCounts []int
-		if objAPI != nil {
-			setDriveCounts = objAPI.SetDriveCounts()
-		}
+
 		globalAPIConfig.init(apiConfig, setDriveCounts)
+
+		// Initialize remote instance transport once.
+		getRemoteInstanceTransportOnce.Do(func() {
+			getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+		})
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 		if err != nil {
 			return fmt.Errorf("Unable to setup Compression: %w", err)
 		}
 		// Validate if the object layer supports compression.
-		if objAPI != nil {
-			if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
-				return fmt.Errorf("Backend does not support compression")
-			}
+		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
 		}
 		globalCompressConfigMu.Lock()
 		globalCompressConfig = cmpCfg
@@ -599,7 +605,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			return fmt.Errorf("Unable to apply scanner config: %w", err)
 		}
 		// update dynamic scanner values.
-		scannerCycle.Update(scannerCfg.Cycle)
+		scannerCycle.Store(scannerCfg.Cycle)
 		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(s, config.LoggerWebhookSubSys)
@@ -609,14 +615,13 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.HTTP {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceIf
+				l.LogOnce = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 				loggerCfg.HTTP[n] = l
 			}
 		}
-		err = logger.UpdateSystemTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
 		}
 	case config.AuditWebhookSubSys:
@@ -627,15 +632,14 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.AuditWebhook {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceIf
+				l.LogOnce = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 				loggerCfg.AuditWebhook[n] = l
 			}
 		}
 
-		err = logger.UpdateAuditWebhookTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
 		}
 	case config.AuditKafkaSubSys:
@@ -649,25 +653,29 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		err = logger.UpdateAuditKafkaTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
 		}
 	case config.StorageClassSubSys:
-		if globalIsErasure && objAPI != nil {
-			setDriveCounts := objAPI.SetDriveCounts()
-			for i, setDriveCount := range setDriveCounts {
-				sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
-				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
-					break
-				}
-				// if we validated all setDriveCounts and it was successful
-				// proceed to store the correct storage class globally.
-				if i == len(setDriveCounts)-1 {
-					globalStorageClass.Update(sc)
-				}
+		for i, setDriveCount := range setDriveCounts {
+			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+				break
 			}
+			// if we validated all setDriveCounts and it was successful
+			// proceed to store the correct storage class globally.
+			if i == len(setDriveCounts)-1 {
+				globalStorageClass.Update(sc)
+			}
+		}
+	case config.CallhomeSubSys:
+		callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
+		} else {
+			globalCallhomeConfig = callhomeCfg
+			updateCallhomeParams(ctx, objAPI)
 		}
 	}
 	globalServerConfigMu.Lock()
@@ -775,13 +783,20 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
+	if globalActiveCred.IsValid() && !globalActiveCred.Equal(auth.DefaultCredentials) {
+		kvs := srvCfg[config.CredentialsSubSys][config.Default]
+		kvs.Set(config.AccessKey, globalActiveCred.AccessKey)
+		kvs.Set(config.SecretKey, globalActiveCred.SecretKey)
+		srvCfg[config.CredentialsSubSys][config.Default] = kvs
+	}
+
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
+	return saveServerConfig(GlobalContext, objAPI, srvCfg)
 }
 
 func getValidConfig(objAPI ObjectLayer) (config.Config, error) {

cmd/config-encrypted.go

@@ -69,8 +69,14 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		return err
 	}
 
+	// If backend doesn't have this file means we have already
+	// attempted then migration
+	if !encrypted {
+		return nil
+	}
+
 	if encrypted && GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat()
+		stat, err := GlobalKMS.Stat(ctx)
 		if err != nil {
 			return err
 		}
@@ -109,6 +115,8 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 						}
 					}
+				} else {
+					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 				}
 			}
 			data = pdata
@@ -131,6 +139,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 	if encrypted && GlobalKMS != nil {
 		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
 	}
+
 	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
 }
 
@@ -139,51 +148,63 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		return nil
 	}
 	if encrypted && GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat()
+		stat, err := GlobalKMS.Stat(context.Background())
 		if err != nil {
 			return err
 		}
 		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 
-	var marker string
-	for {
-		res, err := objAPI.ListObjects(GlobalContext, minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
+	results := make(chan ObjectInfo)
+	if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
+		return err
+	}
+
+	for obj := range results {
+		data, err := readConfig(GlobalContext, objAPI, obj.Name)
 		if err != nil {
 			return err
 		}
-		for _, obj := range res.Objects {
-			data, err := readConfig(GlobalContext, objAPI, obj.Name)
+
+		if !utf8.Valid(data) {
+			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+			if err != nil {
+				if GlobalKMS != nil {
+					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+						minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
+					})
+					if err != nil {
+						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+							minioMetaBucket: obj.Name,
+						})
+						if err != nil {
+							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
+						}
+					}
+				} else {
+					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
+				}
+			}
+			data = pdata
+		}
+
+		if GlobalKMS != nil {
+			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
+				obj.Bucket: path.Join(obj.Bucket, obj.Name),
+			})
 			if err != nil {
 				return err
 			}
-
-			if !utf8.Valid(data) {
-				data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-				if err != nil {
-					return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
-				}
-			}
-			if GlobalKMS != nil {
-				data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-					obj.Bucket: path.Join(obj.Bucket, obj.Name),
-				})
-				if err != nil {
-					return err
-				}
-			}
-			if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
-				return err
-			}
 		}
 
-		if !res.IsTruncated {
-			break
+		if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
+			return err
 		}
-		marker = res.NextMarker
 	}
+
 	if encrypted && GlobalKMS != nil {
 		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
 	}
+
 	return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
 }

cmd/config-migrate_test.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -37,11 +36,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 	globalConfigDir = &ConfigDir{path: rootPath}
 
 	globalObjLayerMutex.Lock()
@@ -51,7 +46,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	// Create a V1 config json file and store it
 	configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
 	configPath := rootPath + "/fsUsers.json"
-	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
+	if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -74,13 +69,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 // Test if all migrate code returns nil when config file does not
 // exist
 func TestServerConfigMigrateInexistentConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	globalConfigDir = &ConfigDir{path: rootPath}
+	globalConfigDir = &ConfigDir{path: t.TempDir()}
 
 	if err := migrateV2ToV3(); err != nil {
 		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -164,11 +153,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 
 // Test if a config migration from v2 to v33 is successfully done
 func TestServerConfigMigrateV2toV33(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 
@@ -181,7 +166,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
+	if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Fire a migrateConfig()
@@ -194,7 +179,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 
 	// Create a V2 config json file and store it
 	configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
-	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
+	if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -234,17 +219,13 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateFaultyConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
+	if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -331,35 +312,31 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	for i := 3; i <= 17; i++ {
 		// Create a corrupted config file
-		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+		if err := os.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
 			0o644); err != nil {
 			t.Fatal("Unexpected error: ", err)
 		}
 
 		// Test different migrate versions and be sure they are returning an error
-		if err = migrateConfig(); err == nil {
+		if err := migrateConfig(); err == nil {
 			t.Fatal("migrateConfig() should fail with a corrupted json")
 		}
 	}
 
 	// Create a corrupted config file for version '2'.
-	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
+	if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test different migrate versions and be sure they are returning an error
-	if err = migrateConfig(); err == nil {
+	if err := migrateConfig(); err == nil {
 		t.Fatal("migrateConfig() should fail with a corrupted json")
 	}
 }

cmd/consolelogger.go

@@ -18,10 +18,11 @@
 package cmd
 
 import (
-	ring "container/ring"
+	"container/ring"
 	"context"
 	"sync"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
@@ -45,7 +46,7 @@ type HTTPConsoleLoggerSys struct {
 // NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
 // the console logging pub sub system
 func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
-	ps := pubsub.New()
+	ps := pubsub.New(8)
 	return &HTTPConsoleLoggerSys{
 		pubsub:  ps,
 		console: console.New(),
@@ -71,11 +72,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
 // HasLogListeners returns true if console log listeners are registered
 // for this node or peers
 func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
-	return sys != nil && sys.pubsub.NumSubscribers() > 0
+	return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
 }
 
 // Subscribe starts console logging for this node.
-func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
 		logger.AddSystemTarget(sys)
@@ -111,11 +112,11 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
 			select {
 			case subCh <- entry:
 			case <-doneCh:
-				return
+				return nil
 			}
 		}
 	}
-	sys.pubsub.Subscribe(subCh, doneCh, filter)
+	return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
 }
 
 // Init if HTTPConsoleLoggerSys is valid, always returns nil right now
@@ -162,9 +163,9 @@ func (sys *HTTPConsoleLoggerSys) Type() types.TargetType {
 
 // Send log message 'e' to console and publish to console
 // log pubsub system
-func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
+func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	var lg log.Info
-	switch e := e.(type) {
+	switch e := entry.(type) {
 	case log.Entry:
 		lg = log.Info{Entry: e, NodeName: sys.nodeName}
 	case string:
@@ -178,5 +179,5 @@ func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
 
-	return sys.console.Send(e, string(logger.All))
+	return sys.console.Send(entry, string(logger.All))
 }

cmd/data-scanner-metric.go

@@ -0,0 +1,315 @@
+package cmd
+
+import (
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+)
+
+//go:generate stringer -type=scannerMetric -trimprefix=scannerMetric $GOFILE
+
+type scannerMetric uint8
+
+type scannerMetrics struct {
+	// All fields must be accessed atomically and aligned.
+	operations [scannerMetricLast]uint64
+	latency    [scannerMetricLastRealtime]lockedLastMinuteLatency
+
+	// actions records actions performed.
+	actions        [lifecycle.ActionCount]uint64
+	actionsLatency [lifecycle.ActionCount]lockedLastMinuteLatency
+
+	// currentPaths contains (string,*currentPathTracker) for each disk processing.
+	// Alignment not required.
+	currentPaths sync.Map
+
+	cycleInfoMu sync.Mutex
+	cycleInfo   *currentScannerCycle
+}
+
+var globalScannerMetrics scannerMetrics
+
+const (
+	// START Realtime metrics, that only to records
+	// last minute latencies and total operation count.
+	scannerMetricReadMetadata scannerMetric = iota
+	scannerMetricCheckMissing
+	scannerMetricSaveUsage
+	scannerMetricApplyAll
+	scannerMetricApplyVersion
+	scannerMetricTierObjSweep
+	scannerMetricHealCheck
+	scannerMetricILM
+	scannerMetricCheckReplication
+	scannerMetricYield
+
+	// START Trace metrics:
+	scannerMetricStartTrace
+	scannerMetricScanObject // Scan object. All operations included.
+
+	// END realtime metrics:
+	scannerMetricLastRealtime
+
+	// Trace only metrics:
+	scannerMetricScanFolder     // Scan a folder on disk, recursively.
+	scannerMetricScanCycle      // Full cycle, cluster global
+	scannerMetricScanBucketDisk // Single bucket on one disk
+
+	// Must be last:
+	scannerMetricLast
+)
+
+// log scanner action.
+// Use for s > scannerMetricStartTrace
+func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].add(duration)
+		}
+
+		if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
+			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " ")))
+		}
+	}
+}
+
+// time a scanner action.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) time(s scannerMetric) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].add(duration)
+		}
+	}
+}
+
+// timeSize add time and size of a scanner action.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) timeSize(s scannerMetric) func(sz int) {
+	startTime := time.Now()
+	return func(sz int) {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].addSize(duration, int64(sz))
+		}
+	}
+}
+
+// incTime will increment time on metric s with a specific duration.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
+	atomic.AddUint64(&p.operations[s], 1)
+	if s < scannerMetricLastRealtime {
+		p.latency[s].add(d)
+	}
+}
+
+func (p *scannerMetrics) incNoTime(s scannerMetric) {
+	atomic.AddUint64(&p.operations[s], 1)
+	if s < scannerMetricLastRealtime {
+		p.latency[s].add(0)
+	}
+}
+
+// timeILM times an ILM action.
+// lifecycle.NoneAction is ignored.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) timeILM(a lifecycle.Action) func() {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return func() {}
+	}
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+		atomic.AddUint64(&p.actions[a], 1)
+		p.actionsLatency[a].add(duration)
+	}
+}
+
+type currentPathTracker struct {
+	name *unsafe.Pointer // contains atomically accessed *string
+}
+
+// currentPathUpdater provides a lightweight update function for keeping track of
+// current objects for each disk.
+// Returns a function that can be used to update the current object
+// and a function to call to when processing finished.
+func (p *scannerMetrics) currentPathUpdater(disk, initial string) (update func(path string), done func()) {
+	initialPtr := unsafe.Pointer(&initial)
+	tracker := &currentPathTracker{
+		name: &initialPtr,
+	}
+
+	p.currentPaths.Store(disk, tracker)
+	return func(path string) {
+			atomic.StorePointer(tracker.name, unsafe.Pointer(&path))
+		}, func() {
+			p.currentPaths.Delete(disk)
+		}
+}
+
+// getCurrentPaths returns the paths currently being processed.
+func (p *scannerMetrics) getCurrentPaths() []string {
+	var res []string
+	prefix := globalLocalNodeName + "/"
+	p.currentPaths.Range(func(key, value interface{}) bool {
+		// We are a bit paranoid, but better miss an entry than crash.
+		name, ok := key.(string)
+		if !ok {
+			return true
+		}
+		obj, ok := value.(*currentPathTracker)
+		if !ok {
+			return true
+		}
+		strptr := (*string)(atomic.LoadPointer(obj.name))
+		if strptr != nil {
+			res = append(res, pathJoin(prefix, name, *strptr))
+		}
+		return true
+	})
+	return res
+}
+
+// activeDisks returns the number of currently active disks.
+// (since this is concurrent it may not be 100% reliable)
+func (p *scannerMetrics) activeDisks() int {
+	var i int
+	p.currentPaths.Range(func(k, v interface{}) bool {
+		i++
+		return true
+	})
+	return i
+}
+
+// lifetime returns the lifetime count of the specified metric.
+func (p *scannerMetrics) lifetime(m scannerMetric) uint64 {
+	if m < 0 || m >= scannerMetricLast {
+		return 0
+	}
+	val := atomic.LoadUint64(&p.operations[m])
+	return val
+}
+
+// lastMinute returns the last minute statistics of a metric.
+// m should be < scannerMetricLastRealtime
+func (p *scannerMetrics) lastMinute(m scannerMetric) AccElem {
+	if m < 0 || m >= scannerMetricLastRealtime {
+		return AccElem{}
+	}
+	val := p.latency[m].total()
+	return val
+}
+
+// lifetimeActions returns the lifetime count of the specified ilm metric.
+func (p *scannerMetrics) lifetimeActions(a lifecycle.Action) uint64 {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return 0
+	}
+	val := atomic.LoadUint64(&p.actions[a])
+	return val
+}
+
+// lastMinuteActions returns the last minute statistics of an ilm metric.
+func (p *scannerMetrics) lastMinuteActions(a lifecycle.Action) AccElem {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return AccElem{}
+	}
+	val := p.actionsLatency[a].total()
+	return val
+}
+
+// setCycle updates the current cycle metrics.
+func (p *scannerMetrics) setCycle(c *currentScannerCycle) {
+	if c != nil {
+		c2 := c.clone()
+		c = &c2
+	}
+	p.cycleInfoMu.Lock()
+	p.cycleInfo = c
+	p.cycleInfoMu.Unlock()
+}
+
+// getCycle returns the current cycle metrics.
+// If not nil, the returned value can safely be modified.
+func (p *scannerMetrics) getCycle() *currentScannerCycle {
+	p.cycleInfoMu.Lock()
+	defer p.cycleInfoMu.Unlock()
+	if p.cycleInfo == nil {
+		return nil
+	}
+	c := p.cycleInfo.clone()
+	return &c
+}
+
+func (p *scannerMetrics) report() madmin.ScannerMetrics {
+	var m madmin.ScannerMetrics
+	cycle := p.getCycle()
+	if cycle != nil {
+		m.CurrentCycle = cycle.current
+		m.CyclesCompletedAt = cycle.cycleCompleted
+		m.CurrentStarted = cycle.started
+	}
+	m.CollectedAt = time.Now()
+	m.ActivePaths = p.getCurrentPaths()
+	m.LifeTimeOps = make(map[string]uint64, scannerMetricLast)
+	for i := scannerMetric(0); i < scannerMetricLast; i++ {
+		if n := atomic.LoadUint64(&p.operations[i]); n > 0 {
+			m.LifeTimeOps[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeOps) == 0 {
+		m.LifeTimeOps = nil
+	}
+
+	m.LastMinute.Actions = make(map[string]madmin.TimedAction, scannerMetricLastRealtime)
+	for i := scannerMetric(0); i < scannerMetricLastRealtime; i++ {
+		lm := p.lastMinute(i)
+		if lm.N > 0 {
+			m.LastMinute.Actions[i.String()] = lm.asTimedAction()
+		}
+	}
+	if len(m.LastMinute.Actions) == 0 {
+		m.LastMinute.Actions = nil
+	}
+
+	// ILM
+	m.LifeTimeILM = make(map[string]uint64)
+	for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ {
+		if n := atomic.LoadUint64(&p.actions[i]); n > 0 {
+			m.LifeTimeILM[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeILM) == 0 {
+		m.LifeTimeILM = nil
+	}
+
+	if len(m.LifeTimeILM) > 0 {
+		m.LastMinute.ILM = make(map[string]madmin.TimedAction, len(m.LifeTimeILM))
+		for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ {
+			lm := p.lastMinuteActions(i)
+			if lm.N > 0 {
+				m.LastMinute.ILM[i.String()] = madmin.TimedAction{Count: uint64(lm.N), AccTime: uint64(lm.Total)}
+			}
+		}
+		if len(m.LastMinute.ILM) == 0 {
+			m.LastMinute.ILM = nil
+		}
+	}
+	return m
+}

cmd/data-scanner.go

@@ -31,7 +31,6 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
@@ -45,15 +44,17 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
+	uatomic "go.uber.org/atomic"
 )
 
 const (
-	dataScannerSleepPerFolder     = time.Millisecond                 // Time to wait between folders.
-	dataUsageUpdateDirCycles      = 16                               // Visit all folders every n cycles.
-	dataScannerCompactLeastObject = 500                              // Compact when there is less than this many objects in a branch.
-	dataScannerCompactAtChildren  = 10000                            // Compact when there are this many children in a branch.
-	dataScannerCompactAtFolders   = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
-	dataScannerStartDelay         = 1 * time.Minute                  // Time to wait on startup and between cycles.
+	dataScannerSleepPerFolder        = time.Millisecond                 // Time to wait between folders.
+	dataUsageUpdateDirCycles         = 16                               // Visit all folders every n cycles.
+	dataScannerCompactLeastObject    = 500                              // Compact when there is less than this many objects in a branch.
+	dataScannerCompactAtChildren     = 10000                            // Compact when there are this many children in a branch.
+	dataScannerCompactAtFolders      = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
+	dataScannerForceCompactAtFolders = 1_000_000                        // Compact when this many subfolders in a single folder (even top level).
+	dataScannerStartDelay            = 1 * time.Minute                  // Time to wait on startup and between cycles.
 
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
@@ -63,12 +64,14 @@ const (
 var (
 	globalHealConfig heal.Config
 
-	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
+		timeout:       30 * time.Second,
+		minimum:       10 * time.Second,
+		retryInterval: time.Second,
+	})
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper = newDynamicSleeper(10, 10*time.Second)
-	scannerCycle   = &safeDuration{
-		t: dataScannerStartDelay,
-	}
+	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
+	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
 )
 
 // initDataScanner will start the scanner in the background.
@@ -78,7 +81,7 @@ func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
 		// Run the data scanner in a loop
 		for {
 			runDataScanner(ctx, objAPI)
-			duration := time.Duration(r.Float64() * float64(scannerCycle.Get()))
+			duration := time.Duration(r.Float64() * float64(scannerCycle.Load()))
 			if duration < time.Second {
 				// Make sure to sleep atleast a second to avoid high CPU ticks.
 				duration = time.Second
@@ -88,23 +91,6 @@ func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
 	}()
 }
 
-type safeDuration struct {
-	sync.Mutex
-	t time.Duration
-}
-
-func (s *safeDuration) Update(t time.Duration) {
-	s.Lock()
-	defer s.Unlock()
-	s.t = t
-}
-
-func (s *safeDuration) Get() time.Duration {
-	s.Lock()
-	defer s.Unlock()
-	return s.t
-}
-
 func getCycleScanMode(currentCycle, bitrotStartCycle uint64, bitrotStartTime time.Time) madmin.HealScanMode {
 	bitrotCycle := globalHealConfig.BitrotScanCycle()
 	switch bitrotCycle {
@@ -180,17 +166,22 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 	// No unlock for "leader" lock.
 
 	// Load current bloom cycle
-	nextBloomCycle := intDataUpdateTracker.current() + 1
+	var cycleInfo currentScannerCycle
+	cycleInfo.next = intDataUpdateTracker.current() + 1
 
 	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
-	if len(buf) >= 8 {
-		if err = binary.Read(bytes.NewReader(buf), binary.LittleEndian, &nextBloomCycle); err != nil {
-			logger.LogIf(ctx, err)
-		}
+	if len(buf) == 8 {
+		cycleInfo.next = binary.LittleEndian.Uint64(buf)
+	} else if len(buf) > 8 {
+		cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
+		buf = buf[8:]
+		_, err = cycleInfo.UnmarshalMsg(buf)
+		logger.LogIf(ctx, err)
 	}
 
-	scannerTimer := time.NewTimer(scannerCycle.Get())
+	scannerTimer := time.NewTimer(scannerCycle.Load())
 	defer scannerTimer.Stop()
+	defer globalScannerMetrics.setCycle(nil)
 
 	for {
 		select {
@@ -198,20 +189,22 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			return
 		case <-scannerTimer.C:
 			// Reset the timer for next cycle.
-			scannerTimer.Reset(scannerCycle.Get())
+			// If scanner takes longer we start at once.
+			scannerTimer.Reset(scannerCycle.Load())
 
-			if intDataUpdateTracker.debug {
-				console.Debugln("starting scanner cycle")
-			}
+			stopFn := globalScannerMetrics.log(scannerMetricScanCycle)
+			cycleInfo.current = cycleInfo.next
+			cycleInfo.started = time.Now()
+			globalScannerMetrics.setCycle(&cycleInfo)
 
 			bgHealInfo := readBackgroundHealInfo(ctx, objAPI)
-			scanMode := getCycleScanMode(nextBloomCycle, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
+			scanMode := getCycleScanMode(cycleInfo.current, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
 			if bgHealInfo.CurrentScanMode != scanMode {
 				newHealInfo := bgHealInfo
 				newHealInfo.CurrentScanMode = scanMode
 				if scanMode == madmin.HealDeepScan {
 					newHealInfo.BitrotStartTime = time.Now().UTC()
-					newHealInfo.BitrotStartCycle = nextBloomCycle
+					newHealInfo.BitrotStartCycle = cycleInfo.current
 				}
 				saveBackgroundHealInfo(ctx, objAPI, newHealInfo)
 			}
@@ -219,18 +212,26 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
+			bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle), scanMode)
+			err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
+			stopFn()
 			if err == nil {
 				// Store new cycle...
-				nextBloomCycle++
-				var tmp [8]byte
-				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				if err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp[:]); err != nil {
-					logger.LogIf(ctx, err)
+				cycleInfo.next++
+				cycleInfo.current = 0
+				cycleInfo.cycleCompleted = append(cycleInfo.cycleCompleted, time.Now())
+				if len(cycleInfo.cycleCompleted) > dataUsageUpdateDirCycles {
+					cycleInfo.cycleCompleted = cycleInfo.cycleCompleted[len(cycleInfo.cycleCompleted)-dataUsageUpdateDirCycles:]
 				}
+				globalScannerMetrics.setCycle(&cycleInfo)
+				tmp := make([]byte, 8, 8+cycleInfo.Msgsize())
+				// Cycle for backward compat.
+				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
+				tmp, _ = cycleInfo.MarshalMsg(tmp)
+				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
+				logger.LogIf(ctx, err)
 			}
 		}
 	}
@@ -262,24 +263,11 @@ type folderScanner struct {
 	// Will not be closed when returned.
 	updates    chan<- dataUsageEntry
 	lastUpdate time.Time
+
+	// updateCurrentPath should be called whenever a new path is scanned.
+	updateCurrentPath func(string)
 }
 
-type scannerStats struct {
-	// All fields must be accessed atomically and aligned.
-
-	accTotalObjects  uint64
-	accTotalVersions uint64
-	accFolders       uint64
-	bucketsStarted   uint64
-	bucketsFinished  uint64
-	ilmChecks        uint64
-
-	// actions records actions performed.
-	actions [lifecycle.ActionCount]uint64
-}
-
-var globalScannerStats scannerStats
-
 // Cache structure and compaction:
 //
 // A cache structure will be kept with a tree of usages.
@@ -323,24 +311,14 @@ var globalScannerStats scannerStats
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
-	t := UTCNow()
-
 	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
-	atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
-	defer func() {
-		atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
-	}()
-	if intDataUpdateTracker.debug {
-		defer func() {
-			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
-		}()
-	}
 
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
 	}
+	updatePath, closeDisk := globalScannerMetrics.currentPathUpdater(basePath, cache.Info.Name)
+	defer closeDisk()
 
 	s := folderScanner{
 		root:                  basePath,
@@ -353,6 +331,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		healObjectSelect:      0,
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
+		updateCurrentPath:     updatePath,
 	}
 
 	// Add disks for set healing.
@@ -384,14 +363,8 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 			s.withFilter = nil
 		}
 	}
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Start scanning. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
-	}
 
 	done := ctx.Done()
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
-	}
 
 	// Read top level in bucket.
 	select {
@@ -407,9 +380,6 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		return cache, err
 	}
 
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Finished scanner, %v entries (%+v) %s \n", len(s.newCache.Cache), *s.newCache.sizeRecursive(s.newCache.Info.Name), logSuffix)
-	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle = cache.Info.NextCycle
 	return s.newCache, nil
@@ -438,10 +408,10 @@ func (f *folderScanner) sendUpdate() {
 func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, into *dataUsageEntry) error {
 	done := ctx.Done()
 	scannerLogPrefix := color.Green("folder-scanner:")
+
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
-	atomic.AddUint64(&globalScannerStats.accFolders, 1)
 
 	for {
 		select {
@@ -601,9 +571,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		}
 
 		// If we have many subfolders, compact ourself.
-		if !into.Compacted &&
-			f.newCache.Info.Name != folder.name &&
-			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders {
+		shouldCompact := f.newCache.Info.Name != folder.name &&
+			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
+			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
+
+		if !into.Compacted && shouldCompact {
 			into.Compacted = true
 			newFolders = append(newFolders, existingFolders...)
 			existingFolders = nil
@@ -666,7 +638,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					f.updateCache.replaceHashed(h, &thisHash, dataUsageEntry{})
 				}
 			}
+			f.updateCurrentPath(folder.name)
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
+			stopFn()
 			// Add new folders if this is new and we don't have existing.
 			if !into.Compacted {
 				parent := f.updateCache.find(thisHash.Key())
@@ -694,7 +669,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
+			f.updateCurrentPath(folder.name)
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name, "EXISTING")
 			scanFolder(folder)
+			stopFn()
 		}
 
 		// Scan for healing
@@ -735,9 +713,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		healObjectsPrefix := color.Green("healObjects:")
 		for k := range abandonedChildren {
 			bucket, prefix := path2BucketObject(k)
-			if f.dataUsageScannerDebug {
-				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
-			}
+			stopFn := globalScannerMetrics.time(scannerMetricCheckMissing)
+			f.updateCurrentPath(k)
 
 			if bucket != resolver.bucket {
 				// Bucket might be missing as well with abandoned children.
@@ -764,11 +741,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 				},
 				// Some disks have data for this.
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-					if f.dataUsageScannerDebug {
-						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
-					}
-
+				partial: func(entries metaCacheEntries, errs []error) {
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						// check if we can get one entry atleast
@@ -825,6 +798,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				},
 			})
 
+			stopFn()
 			if f.dataUsageScannerDebug && err != nil && err != errFileNotFound {
 				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
@@ -832,7 +806,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// Add unless healing returned an error.
 			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
+				stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED")
 				scanFolder(this)
+				stopFn()
 			}
 		}
 		break
@@ -982,7 +958,6 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 		return false, size
 	}
 
-	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
 	action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
@@ -993,7 +968,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
 		}
 	}
-	atomic.AddUint64(&globalScannerStats.actions[action], 1)
+	defer globalScannerMetrics.timeILM(action)
 
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1057,9 +1032,13 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 	fivs = fivs[:lim+1]
 
 	rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
+	vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
+
+	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
+
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
-		obj := fi.ToObjectInfo(i.bucket, i.objectPath())
+		obj := fi.ToObjectInfo(i.bucket, i.objectPath(), versioned)
 		// skip versions with object locking enabled
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			if i.debug {
@@ -1106,16 +1085,23 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 // The metadata will be compared to consensus on the object layer before any changes are applied.
 // If no metadata is supplied, -1 is returned if no action is taken.
 func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) int64 {
+	done := globalScannerMetrics.time(scannerMetricILM)
 	applied, size := i.applyLifecycle(ctx, o, oi)
+	done()
+
 	// For instance, an applied lifecycle means we remove/transitioned an object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
 		if i.heal.enabled {
+			done := globalScannerMetrics.time(scannerMetricHealCheck)
 			size = i.applyHealing(ctx, o, oi)
+			done()
 		}
 		// replicate only if lifecycle rules are not applied.
+		done := globalScannerMetrics.time(scannerMetricCheckReplication)
 		i.healReplication(ctx, o, oi.Clone(), sizeS)
+		done()
 	}
 	return size
 }
@@ -1184,7 +1170,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		opts.VersionID = obj.VersionID
 	}
 	if opts.VersionID == "" {
-		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
+		opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
+		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 	}
 
 	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
@@ -1242,25 +1229,15 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
-	roi := getHealReplicateObjectInfo(oi, i.replication)
-
-	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
-		// heal delete marker replication failure or versioned delete replication failure
-		if oi.ReplicationStatus == replication.Pending ||
-			oi.ReplicationStatus == replication.Failed ||
-			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, roi)
-			return
-		}
-		// if replication status is Complete on DeleteMarker and existing object resync required
-		if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
-			i.healReplicationDeletes(ctx, o, roi)
-			return
-		}
+	if oi.VersionID == "" {
 		return
 	}
-	if roi.ExistingObjResync.mustResync() {
-		roi.OpType = replication.ExistingObjectReplicationType
+	if i.replication.Config == nil {
+		return
+	}
+	roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication)
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+		return
 	}
 
 	if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 {
@@ -1291,48 +1268,9 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 	}
 
 	switch oi.ReplicationStatus {
-	case replication.Pending, replication.Failed:
-		globalReplicationPool.queueReplicaTask(roi)
-		return
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
-	if roi.ExistingObjResync.mustResync() {
-		globalReplicationPool.queueReplicaTask(roi)
-	}
-}
-
-// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
-	// handle soft delete and permanent delete failures here.
-	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
-		versionID := ""
-		dmVersionID := ""
-		if roi.VersionPurgeStatus.Empty() {
-			dmVersionID = roi.VersionID
-		} else {
-			versionID = roi.VersionID
-		}
-
-		doi := DeletedObjectReplicationInfo{
-			DeletedObject: DeletedObject{
-				ObjectName:            roi.Name,
-				DeleteMarkerVersionID: dmVersionID,
-				VersionID:             versionID,
-				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-				DeleteMarker:          roi.DeleteMarker,
-			},
-			Bucket: roi.Bucket,
-			OpType: replication.HealReplicationType,
-		}
-		if roi.ExistingObjResync.mustResync() {
-			doi.OpType = replication.ExistingObjectReplicationType
-			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
-			return
-		}
-		globalReplicationPool.queueReplicaDeleteTask(doi)
-	}
 }
 
 type dynamicSleeper struct {
@@ -1351,15 +1289,20 @@ type dynamicSleeper struct {
 
 	// cycle will be closed
 	cycle chan struct{}
+
+	// isScanner should be set when this is used by the scanner
+	// to record metrics.
+	isScanner bool
 }
 
 // newDynamicSleeper
-func newDynamicSleeper(factor float64, maxWait time.Duration) *dynamicSleeper {
+func newDynamicSleeper(factor float64, maxWait time.Duration, isScanner bool) *dynamicSleeper {
 	return &dynamicSleeper{
-		factor:   factor,
-		cycle:    make(chan struct{}),
-		maxSleep: maxWait,
-		minSleep: 100 * time.Microsecond,
+		factor:    factor,
+		cycle:     make(chan struct{}),
+		maxSleep:  maxWait,
+		minSleep:  100 * time.Microsecond,
+		isScanner: isScanner,
 	}
 }
 
@@ -1389,15 +1332,24 @@ func (d *dynamicSleeper) Timer(ctx context.Context) func() {
 			select {
 			case <-ctx.Done():
 				if !timer.Stop() {
+					if d.isScanner {
+						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+					}
 					<-timer.C
 				}
 				return
 			case <-timer.C:
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			case <-cycle:
 				if !timer.Stop() {
 					// We expired.
 					<-timer.C
+					if d.isScanner {
+						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+					}
 					return
 				}
 			}
@@ -1428,14 +1380,23 @@ func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
 		case <-ctx.Done():
 			if !timer.Stop() {
 				<-timer.C
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 			}
 			return
 		case <-timer.C:
+			if d.isScanner {
+				globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+			}
 			return
 		case <-cycle:
 			if !timer.Stop() {
 				// We expired.
 				<-timer.C
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			}
 		}
@@ -1467,9 +1428,9 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 	var apiName string
-	switch trigger {
+	switch event {
 	case ILMExpiry:
 		apiName = "ILMExpiry"
 	case ILMFreeVersionDelete:
@@ -1478,7 +1439,7 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
 		apiName = "ILMTransition"
 	}
 	auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
-		Trigger:   trigger,
+		Event:     event,
 		APIName:   apiName,
 		VersionID: oi.VersionID,
 	})