fix: update storage-class properly

fixes #14005

.github/workflows/go-cross.yml

@@ -1,23 +1,26 @@
-name: Go
+name: Crosscompile
 
 on:
   pull_request:
     branches:
     - master
 
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
-    name: MinIO crosscompile tests on ${{ matrix.go-version }} and ${{ matrix.os }}
+    name: Build Tests with Go ${{ matrix.go-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x]
+        go-version: [1.17.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
-        with:
-          node-version: '12'
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/go-healing.yml

@@ -0,0 +1,48 @@
+name: Healing Functional Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
+          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
+          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
+          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_AUTO_ENCRYPTION: on
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make verify-healing

.github/workflows/go-lint.yml

@@ -1,23 +1,47 @@
-name: Go
+name: Linters and Tests
 
 on:
   pull_request:
     branches:
     - master
 
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
-    name: MinIO tests on ${{ matrix.go-version }} and ${{ matrix.os }}
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x]
+        go-version: [1.17.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        if: matrix.os == 'ubuntu-latest'
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - uses: actions/cache@v2
+        if: matrix.os == 'windows-latest'
+        with:
+          path: |
+            %LocalAppData%\go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'windows-latest'
         env:
@@ -39,4 +63,5 @@ jobs:
           curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
           go list -deps -json ./... | jq -s 'unique_by(.Module.Path)|.[]|select(has("Module"))|.Module' | ./nancy sleuth
           make
+          make test
           make test-race

.github/workflows/go.yml

@@ -1,23 +1,37 @@
-name: Go
+name: Functional Tests
 
 on:
   pull_request:
     branches:
     - master
 
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
-    name: MinIO Setup on ${{ matrix.go-version }} and ${{ matrix.os }}
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.16.x, 1.17.x]
+        go-version: [1.17.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:
@@ -32,4 +46,3 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify
-          make verify-healing

.github/workflows/iam-integrations.yaml

@@ -0,0 +1,91 @@
+name: IAM integration
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  iam-matrix-test:
+    name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]"
+    runs-on: ubuntu-latest
+
+    services:
+      openldap:
+        image: quay.io/minio/openldap
+        ports:
+          - "389:389"
+          - "636:636"
+        env:
+          LDAP_ORGANIZATION: "MinIO Inc"
+          LDAP_DOMAIN: "min.io"
+          LDAP_ADMIN_PASSWORD: "admin"
+      etcd:
+        image: "quay.io/coreos/etcd:v3.5.1"
+        env:
+          ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379"
+          ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379"
+        ports:
+          - "2379:2379"
+        options: >-
+          --health-cmd "etcdctl endpoint health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      openid:
+        image: quay.io/minio/dex
+        ports:
+          - "5556:5556"
+        env:
+          DEX_LDAP_SERVER: "openldap:389"
+
+    strategy:
+      # When ldap, etcd or openid vars are empty below, those external servers
+      # are turned off - i.e. if ldap="", then ldap server is not enabled for
+      # the tests.
+      matrix:
+        go-version: [1.17.x]
+        ldap: ["", "localhost:389"]
+        etcd: ["", "http://localhost:2379"]
+        openid: ["", "http://127.0.0.1:5556/dex"]
+        exclude:
+          # exclude combos where all are empty.
+          - ldap: ""
+            etcd: ""
+            openid: ""
+          # exclude combos where both ldap and openid IDPs are specified.
+          - ldap: "localhost:389"
+            openid: "http://127.0.0.1:5556/dex"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Test LDAP/OpenID/Etcd combo
+        env:
+          LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          ETCD_SERVER: ${{ matrix.etcd }}
+          OPENID_TEST_SERVER: ${{ matrix.openid }}
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-iam
+      - name: Test LDAP for automatic site replication
+        if: matrix.ldap == 'localhost:389'
+        run: |
+          make test-site-replication

.github/workflows/replication.yaml

@@ -0,0 +1,40 @@
+name: Multi-site replication tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  replication-test:
+    name: Replication Tests with Go ${{ matrix.go-version }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Test Replication
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-replication

.github/workflows/upgrade-ci-cd.yaml

@@ -0,0 +1,31 @@
+name: Upgrade old version tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Start upgrade tests
+        run: |
+          make test-upgrade

.gitignore

@@ -22,3 +22,14 @@ stage/
 .sia_temp/
 config.json
 node_modules/
+mc.*
+s3-check-md5*
+xl-meta*
+healing-*
+inspect*
+200M*
+hash-set
+minio.RELEASE*
+mc
+nancy
+inspects/*

.golangci.yml

@@ -23,12 +23,19 @@ linters:
     - structcheck
     - unconvert
     - varcheck
+    - gocritic
 
 issues:
   exclude-use-default: false
   exclude:
       - should have a package comment
       - error strings should not be capitalized or end with punctuation or a newline
+      # todo fix these when we get enough time.
+      - "singleCaseSwitch: should rewrite switch statement to if statement"
+      - "unlambda: replace"
+      - "captLocal:"
+      - "ifElseChain:"
+      - "elseif:"
 
 service:
-  golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.43.0 # use the fixed version to not introduce new linters unexpectedly

COMPLIANCE.md

@@ -0,0 +1,6 @@
+# AGPLv3 Compliance
+We have designed MinIO as an Open Source software for the Open Source software community. This requires applications to consider whether their usage of MinIO is in compliance with the GNU AGPLv3 [license](https://github.com/minio/minio/blob/master/LICENSE).
+
+MinIO cannot make the determination as to whether your application's usage of MinIO is in compliance with the AGPLv3 license requirements. You should instead rely on your own legal counsel or licensing specialists to audit and ensure your application is in compliance with the licenses of MinIO and all other open-source projects with which your application integrates or interacts. We understand that AGPLv3 licensing is complex and nuanced. It is for that reason we strongly encourage using experts in licensing to make any such determinations around compliance instead of relying on apocryphal or anecdotal advice.
+
+[MinIO Commercial Licensing](https://min.io/pricing) is the best option for applications that trigger AGPLv3 obligations (e.g. open sourcing your application). Applications using MinIO - or any other OSS-licensed code - without validating their usage do so at their own risk.

Dockerfile

@@ -1,5 +1,8 @@
 FROM minio/minio:latest
 
+COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]

Dockerfile.dev

@@ -1,14 +1,7 @@
-FROM minio/minio:edge
+FROM minio/minio:latest
 
-LABEL maintainer="MinIO Inc <dev@min.io>"
+COPY ./minio /opt/bin/minio
-
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-COPY minio /usr/bin/
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
-
-RUN chmod +x /usr/bin/minio && \
-    chmod +x /usr/bin/docker-entrypoint.sh
-
-EXPOSE 9000
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Dockerfile.hotfix

@@ -0,0 +1,50 @@
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
+
+ARG RELEASE
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=/opt/bin:$PATH
+
+COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
+
+RUN \
+     microdnf clean all && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+     microdnf install minisign --nodocs && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
+     microdnf clean all && \
+     chmod +x /opt/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/verify-minio.sh && \
+     /usr/bin/verify-minio.sh && \
+     microdnf clean all
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.release

@@ -18,7 +18,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=/opt/bin:$PATH
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
@@ -26,18 +27,21 @@ COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
 
 RUN \
+     microdnf clean all && \
      microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /usr/bin/minio && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
      microdnf clean all && \
-     chmod +x /usr/bin/minio && \
+     chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh
+     /usr/bin/verify-minio.sh && \
+     microdnf clean all
 
 EXPOSE 9000
 

Dockerfile.release.fips

@@ -18,7 +18,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=/opt/bin:$PATH
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
@@ -26,18 +27,21 @@ COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
 
 RUN \
+     microdnf clean all && \
      microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /usr/bin/minio && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
      microdnf clean all && \
-     chmod +x /usr/bin/minio && \
+     chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh
+     /usr/bin/verify-minio.sh && \
+     microdnf clean all
 
 EXPOSE 9000
 

Dockerfile.scratch

@@ -0,0 +1,5 @@
+FROM scratch
+
+COPY minio /minio
+
+CMD ["/minio"]

Makefile

@@ -10,78 +10,107 @@ TAG ?= "minio/minio:$(VERSION)"
 
 all: build
 
-checks:
+checks: ## check dependencies
 	@echo "Checking dependencies"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
-getdeps:
+help: ## print this help
-	@mkdir -p ${GOPATH}/bin
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.40.1
-	@which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3)
-	@which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer)
 
-crosscompile:
+getdeps: ## fetch necessary dependencies
+	@mkdir -p ${GOPATH}/bin
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
+	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
+
+crosscompile: ## cross compile minio
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
 verifiers: getdeps lint check-gen
 
-check-gen:
+check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
-lint:
+lint: ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
 
-# Builds minio, runs the verifiers then runs the tests.
 check: test
-test: verifiers build
+test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@GOGC=25 GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./...
 
-test-race: verifiers build
+test-upgrade: build
+	@echo "Running minio upgrade tests"
+	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)
+
+test-race: verifiers build ## builds minio, runs linters, tests (race)
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-# Verify minio binary
+test-iam: build ## verify IAM (external IDP, etcd backends)
-verify:
+	@echo "Running tests for IAM (external IDP, etcd backends)"
+	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
+	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
+	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
+
+test-replication: install ## verify multi site replication
+	@echo "Running tests for replicating three sites"
+	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
+
+test-site-replication: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site.sh)
+
+verify: ## verify minio various setups
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-# Verify healing of disks with minio binary
+verify-healing: ## verify healing and replacing disks with minio binary
-verify-healing:
 	@echo "Verify healing build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
+	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
-# Builds minio locally.
+build: checks ## builds minio to $(PWD)
-build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
-	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
+	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
-hotfix: hotfix-vars install
+	$(eval TAG := "minio/minio:$(VERSION)")
 
-docker-hotfix: hotfix checks
+hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+	@mv -f ./minio ./minio.$(VERSION)
+	@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
+
+hotfix-push: hotfix
+	@scp -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+
+docker-hotfix-push: docker-hotfix
+	@docker push $(TAG)
+
+docker-hotfix: hotfix-push checks ## builds minio docker container with hotfix tags
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile.dev
+	@docker build -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix
 
-docker: build checks
+docker: build checks ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile.dev
+	@docker build -t $(TAG) . -f Dockerfile
 
-# Builds minio and installs it to $GOPATH/bin.
+install: build ## builds minio and installs it to $GOPATH/bin.
-install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
-clean:
+clean: ## cleanup all generated assets
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@find . -name '*~' | xargs rm -fv

README.md

@@ -21,10 +21,8 @@ for more complete documentation.
 Run the following command to run the latest stable image of MinIO as a container using an ephemeral data volume:
 
 ```sh
-podman run \
+podman run -p 9000:9000 -p 9001:9001 \
-  -p 9000:9000 \
+  quay.io/minio/minio server /data --console-address ":9001"
-  -p 9001:9001 \
-  minio/minio server /data --console-address ":9001"
 ```
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded
@@ -126,7 +124,7 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 
 # Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.16](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go install github.com/minio/minio@latest
@@ -239,13 +237,11 @@ mc admin update <minio alias, e.g., myminio>
 
 ## Important things to remember during MinIO upgrades
 
+- Container environments are advised to update the container images instead of updating binaries inside the container.
+- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
-- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
-- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
 - If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
-- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
-- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
 # Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
@@ -259,5 +255,6 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 # License
-MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file. 
+- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
-MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)

buildscripts/minio-upgrade.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+trap 'cleanup $LINENO' ERR
+
+# shellcheck disable=SC2120
+cleanup() {
+    MINIO_VERSION=dev docker-compose \
+                 -f "buildscripts/upgrade-tests/compose.yml" \
+                 rm -s -f
+    docker volume prune -f
+}
+
+verify_checksum_after_heal() {
+    local sum1
+    sum1=$(curl -s "$2" | sha256sum);
+    mc admin heal --json -r "$1" >/dev/null; # test after healing
+    local sum1_heal
+    sum1_heal=$(curl -s "$2" | sha256sum);
+
+    if [ "${sum1_heal}" != "${sum1}" ]; then
+        echo "mismatch expected ${sum1_heal}, got ${sum1}"
+        exit 1;
+    fi
+}
+
+verify_checksum_mc() {
+    local expected
+    expected=$(mc cat "$1" | sha256sum)
+    local got
+    got=$(mc cat "$2" | sha256sum)
+
+    if [ "${expected}" != "${got}" ]; then
+        echo "mismatch - expected ${expected}, got ${got}"
+        exit 1;
+    fi
+    echo "matches - ${expected}, got ${got}"
+}
+
+add_alias() {
+    for i in $(seq 1 4); do
+        echo "... attempting to add alias $i"
+        until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
+            echo "...waiting... for 5secs" && sleep 5
+        done
+    done
+
+    echo "Sleeping for nginx"
+    sleep 20
+}
+
+__init__() {
+    sudo apt install curl -y
+    export GOPATH=/tmp/gopath
+    export PATH=${PATH}:${GOPATH}/bin
+
+    go install github.com/minio/mc@latest
+
+    TAG=minio/minio:dev make docker
+
+    MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
+                 -f "buildscripts/upgrade-tests/compose.yml" \
+                 up -d --build
+
+    add_alias
+
+    mc mb minio/minio-test/
+    mc cp ./minio minio/minio-test/to-read/
+    mc cp /etc/hosts minio/minio-test/to-read/hosts
+    mc policy set download minio/minio-test
+
+    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+
+    curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
+
+    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
+}
+
+main() {
+    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
+
+    add_alias
+
+    verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts
+
+    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+
+    verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts
+
+    cleanup
+}
+
+( __init__ "$@" && main "$@" )

buildscripts/unaligned-healing.sh

@@ -0,0 +1,171 @@
+#!/bin/bash -e
+#
+
+set -E
+set -o pipefail
+set -x
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
+	chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
+    fi
+}
+
+function start_minio_16drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
+    "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
+    "${WORK_DIR}/mc" cp \
+		     "${WORK_DIR}/unaligned" \
+		     minio/healing-shard-bucket/unaligned \
+		     --disable-multipart --quiet
+
+    ## "unaligned" object name gets consistently distributed
+    ## to disks in following distribution order
+    ##
+    ## NOTE: if you change the name make sure to change the
+    ## distribution order present here
+    ##
+    ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+
+    ## make sure to remove the "last" data shard
+    rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
+    sleep 10
+    ## Heal the shard
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+    ## then remove any other data shard let's pick first disk
+    ## - 1st data shard.
+    rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
+    sleep 10
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_16drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/upgrade-tests/compose.yml

@@ -0,0 +1,81 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${MINIO_VERSION}
+  command: server http://minio{1...4}/data{1...3}
+  env_file:
+    - ./minio.env
+  expose:
+    - "9000"
+    - "9001"
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - data1-1:/data1
+      - data1-2:/data2
+      - data1-3:/data3
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - data2-1:/data1
+      - data2-2:/data2
+      - data2-3:/data3
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - data3-1:/data1
+      - data3-2:/data2
+      - data3-3:/data3
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - data4-1:/data1
+      - data4-2:/data2
+      - data4-3:/data3
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    ports:
+    - "9000:9000"
+    - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  data1-1:
+  data1-2:
+  data1-3:
+  data2-1:
+  data2-2:
+  data2-3:
+  data3-1:
+  data3-2:
+  data3-3:
+  data4-1:
+  data4-2:
+  data4-3:

buildscripts/upgrade-tests/minio.env

@@ -0,0 +1,3 @@
+MINIO_ACCESS_KEY=minioadmin
+MINIO_SECRET_KEY=minioadmin
+MINIO_BROWSER=off

buildscripts/upgrade-tests/nginx.conf

@@ -0,0 +1,68 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    # main minio
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+         # To allow special characters in headers
+         ignore_invalid_headers off;
+         # Allow any size file to be uploaded.
+         # Set to a value such as 1000m; to restrict file size to a specific value
+         client_max_body_size 0;
+         # To disable buffering
+         proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

buildscripts/verify-build.sh

@@ -20,6 +20,7 @@ export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
 export GOGC=25
+export ENABLE_ADMIN=1
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
@@ -65,7 +66,7 @@ function start_minio_pool_erasure_sets_ipv6()
 {
     export MINIO_ROOT_USER=$ACCESS_KEY
     export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
     "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
     "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 

buildscripts/verify-healing.sh

@@ -18,31 +18,50 @@ function start_minio_3_node() {
     export MINIO_ROOT_PASSWORD=minio123
     export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
-    start_port=$(shuf -i 10000-65000 -n 1)
+    start_port=$2
     args=""
     for i in $(seq 1 3); do
 	args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
     "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    disown $!
+    pid1=$!
+    disown ${pid1}
 
     "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    disown $!
+    pid2=$!
+    disown $pid2
 
     "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    disown $!
+    pid3=$!
+    disown $pid3
 
     sleep "$1"
-    if [ "$(pgrep -c minio)" -ne 3 ]; then
+
-        for i in $(seq 1 3); do
+    if ! ps -p $pid1 1>&2 > /dev/null; then
-            echo "server$i log:"
+	echo "server1 log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
+	cat "${WORK_DIR}/dist-minio-server1.log"
-        done
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
     fi
+
+    if ! ps -p $pid2 1>&2 > /dev/null; then
+	echo "server2 log:"
+	cat "${WORK_DIR}/dist-minio-server2.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ps -p $pid3 1>&2 > /dev/null; then
+	echo "server3 log:"
+	cat "${WORK_DIR}/dist-minio-server3.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
     if ! pkill minio; then
 	for i in $(seq 1 3); do
 	    echo "server$i log:"
@@ -85,14 +104,14 @@ function __init__()
 }
 
 function perform_test() {
-    start_minio_3_node 120
+    start_minio_3_node 120 $2
 
     echo "Testing Distributed Erasure setup healing of drives"
     echo "Remove the contents of the disks belonging to '${1}' erasure set"
 
     rm -rf ${WORK_DIR}/${1}/*/
 
-    start_minio_3_node 120
+    start_minio_3_node 120 $2
 
     rv=$(check_online)
     if [ "$rv" == "1" ]; then
@@ -109,9 +128,12 @@ function perform_test() {
 
 function main()
 {
-    perform_test "2"
+    # use same ports for all tests
-    perform_test "1"
+    start_port=$(shuf -i 10000-65000 -n 1)
-    perform_test "3"
+
+    perform_test "2" ${start_port}
+    perform_test "1" ${start_port}
+    perform_test "3" ${start_port}
 }
 
 ( __init__ "$@" && main "$@" )

cmd/acl-handlers.go

@@ -114,8 +114,6 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
 }
 
 // GetBucketACLHandler - GET Bucket ACL
@@ -164,8 +162,6 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
 }
 
 // PutObjectACLHandler - PUT Object ACL
@@ -229,8 +225,6 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
 }
 
 // GetObjectACLHandler - GET Object ACL
@@ -283,6 +277,4 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
 }

cmd/admin-handler-utils.go

@@ -29,7 +29,7 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
-func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
+func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
@@ -37,15 +37,18 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 		return nil, auth.Credentials{}
 	}
 
+	for _, action := range actions {
 		// Validate request signature.
 		cred, adminAPIErr := checkAdminRequestAuth(ctx, r, action, "")
 		if adminAPIErr != ErrNone {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 			return nil, cred
 		}
-
 		return objectAPI, cred
 	}
+	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+	return nil, auth.Credentials{}
+}
 
 // AdminError - is a generic error for all admin APIs.
 type AdminError struct {
@@ -83,6 +86,8 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			Description:    e.Message,
 			HTTPStatusCode: e.StatusCode,
 		}
+	case SRError:
+		apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause)
 	default:
 		switch {
 		case errors.Is(err, errConfigNotFound):
@@ -103,6 +108,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusServiceUnavailable,
 			}
+		case errors.Is(err, errPolicyInUse):
+			apiErr = APIError{
+				Code:           "XMinioAdminPolicyInUse",
+				Description:    "The policy cannot be removed, as it is in use",
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, kes.ErrKeyExists):
 			apiErr = APIError{
 				Code:           "XMinioKMSKeyExists",

cmd/admin-handlers-config-kv.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -78,6 +79,22 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	dynamic := config.SubSystemsDynamic.Contains(string(kvBytes))
+	if dynamic {
+		applyDynamic(ctx, objectAPI, cfg, r, w)
+	}
+}
+
+func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config, r *http.Request, w http.ResponseWriter) {
+	// Apply dynamic values.
+	if err := applyDynamicConfig(GlobalContext, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	globalNotificationSys.SignalService(serviceReloadDynamic)
+	// Tell the client that dynamic config was applied.
+	w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 }
 
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
@@ -117,7 +134,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
+	if err = validateConfig(cfg); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -135,14 +152,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	if dynamic {
-		// Apply dynamic values.
+		applyDynamic(ctx, objectAPI, cfg, r, w)
-		if err := applyDynamicConfig(GlobalContext, objectAPI, cfg); err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		globalNotificationSys.SignalService(serviceReloadDynamic)
-		// If all values were dynamic, tell the client.
-		w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 	}
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -205,13 +215,11 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 				return
 			}
 		}
-	} else {
+	} else if err := delServerConfigHistory(ctx, objectAPI, restoreID); err != nil {
-		if err := delServerConfigHistory(ctx, objectAPI, restoreID); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
-}
 
 // RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
@@ -248,7 +256,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
+	if err = validateConfig(cfg); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -326,7 +334,6 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	json.NewEncoder(w).Encode(rd)
-	w.(http.Flusher).Flush()
 }
 
 // SetConfigHandler - PUT /minio/admin/v3/config
@@ -360,7 +367,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
+	if err = validateConfig(cfg); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -417,6 +424,8 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 				off = !openid.Enabled(kv)
 			case config.IdentityLDAPSubSys:
 				off = !xldap.Enabled(kv)
+			case config.IdentityTLSSubSys:
+				off = !globalSTSTLSConfig.Enabled
 			}
 			if off {
 				s.WriteString(config.KvComment)

cmd/admin-handlers-site-replication.go

@@ -0,0 +1,313 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
+func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationAdd")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var sites []madmin.PeerSite
+	err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
+//
+// used internally to tell current cluster to enable SR with
+// the provided peer clusters and service account.
+func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerJoin")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var joinArg madmin.SRPeerJoinReq
+	if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
+func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerBucketOps")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	operation := madmin.BktOp(vars["operation"])
+
+	var err error
+	switch operation {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
+	case madmin.MakeWithVersioningBktOp:
+		_, isLockEnabled := r.Form["lockEnabled"]
+		_, isVersioningEnabled := r.Form["versioningEnabled"]
+		opts := BucketOptions{
+			Location:          r.Form.Get("location"),
+			LockEnabled:       isLockEnabled,
+			VersioningEnabled: isVersioningEnabled,
+		}
+		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
+	case madmin.ConfigureReplBktOp:
+		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
+	case madmin.DeleteBucketBktOp:
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+	case madmin.ForceDeleteBucketBktOp:
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+}
+
+// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
+func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateIAMItem")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var item madmin.SRIAMItem
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	var err error
+	switch item.Type {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
+	case madmin.SRIAMItemPolicy:
+		if item.Policy == nil {
+			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+		} else {
+			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
+			if perr != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
+				return
+			}
+			if policy.IsEmpty() {
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+			} else {
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+			}
+		}
+	case madmin.SRIAMItemSvcAcc:
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange)
+	case madmin.SRIAMItemPolicyMapping:
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
+	case madmin.SRIAMItemSTSAcc:
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateBucketItem")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var item madmin.SRBucketMeta
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	var err error
+	switch item.Type {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
+	case madmin.SRBucketMetaTypePolicy:
+		if item.Policy == nil {
+			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+		} else {
+			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
+			if berr != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
+				return
+			}
+			if bktPolicy.IsEmpty() {
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+			} else {
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+			}
+		}
+	case madmin.SRBucketMetaTypeTags:
+		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
+	case madmin.SRBucketMetaTypeObjectLockConfig:
+		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
+	case madmin.SRBucketMetaTypeSSEConfig:
+		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SiteReplicationDisable - PUT /minio/admin/v3/site-replication/disable
+func (a adminAPIHandlers) SiteReplicationDisable(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationDisable")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationDisableAction)
+	if objectAPI == nil {
+		return
+	}
+}
+
+// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
+func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationInfo")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
+	}
+
+	info, err := globalSiteReplicationSys.GetClusterInfo(ctx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	idpSettings := globalSiteReplicationSys.GetIDPSettings(ctx)
+	if err := json.NewEncoder(w).Encode(idpSettings); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		return SRError{
+			Cause: err,
+			Code:  ErrSiteReplicationInvalidRequest,
+		}
+	}
+
+	if encryptionKey != "" {
+		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return SRError{
+				Cause: err,
+				Code:  ErrSiteReplicationInvalidRequest,
+			}
+		}
+	}
+
+	return json.Unmarshal(data, v)
+}

cmd/admin-handlers-users-race_test.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !race
+// +build !race
+
+// Tests in this file are not run under the `-race` flag as they are too slow
+// and cause context deadline errors.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go"
+	minio "github.com/minio/minio-go/v7"
+)
+
+func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
+	suite.SetUpSuite(c)
+	suite.TestDeleteUserRace(c)
+	suite.TearDownSuite(c)
+}
+
+func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) {
+	baseTestCases := []TestSuiteCommon{
+		// Init and run test on FS backend with signature v4.
+		{serverType: "FS", signer: signerV4},
+		// Init and run test on FS backend, with tls enabled.
+		{serverType: "FS", signer: signerV4, secure: true},
+		// Init and run test on Erasure backend.
+		{serverType: "Erasure", signer: signerV4},
+		// Init and run test on ErasureSet backend.
+		{serverType: "ErasureSet", signer: signerV4},
+	}
+	testCases := []*TestSuiteIAM{}
+	for _, bt := range baseTestCases {
+		testCases = append(testCases,
+			newTestSuiteIAM(bt, false),
+			newTestSuiteIAM(bt, true),
+		)
+	}
+	for i, testCase := range testCases {
+		etcdStr := ""
+		if testCase.withEtcdBackend {
+			etcdStr = " (with etcd backend)"
+		}
+		t.Run(
+			fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr),
+			func(t *testing.T) {
+				runAllIAMConcurrencyTests(testCase, &check{t, testCase.serverType})
+			},
+		)
+	}
+}
+
+func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create a policy policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	userCount := 50
+	accessKeys := make([]string, userCount)
+	secretKeys := make([]string, userCount)
+	for i := 0; i < userCount; i++ {
+		accessKey, secretKey := mustGenerateCredentials(c)
+		err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+		if err != nil {
+			c.Fatalf("Unable to set user: %v", err)
+		}
+
+		err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+		if err != nil {
+			c.Fatalf("Unable to set policy: %v", err)
+		}
+
+		accessKeys[i] = accessKey
+		secretKeys[i] = secretKey
+	}
+
+	wg := sync.WaitGroup{}
+	for i := 0; i < userCount; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
+			err := s.adm.RemoveUser(ctx, accessKeys[i])
+			if err != nil {
+				c.Fatalf("unable to remove user: %v", err)
+			}
+			c.mustNotListObjects(ctx, uClient, bucket)
+		}(i)
+	}
+	wg.Wait()
+}

cmd/admin-handlers-users.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -57,18 +58,10 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to delete user.
-	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // ListUsers - GET /minio/admin/v3/list-users?bucket={bucket}
@@ -163,34 +156,27 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	accessKey := cred.ParentUser
+	checkDenyOnly := false
-	if accessKey == "" {
+	if name == cred.AccessKey {
-		accessKey = cred.AccessKey
+		// Check that there is no explicit deny - otherwise it's allowed
+		// to view one's own info.
+		checkDenyOnly = true
 	}
 
-	// For temporary credentials always
-	// the temporary credentials to check
-	// policy without implicit permissions.
-	if cred.IsTemp() && cred.ParentUser == globalActiveCred.AccessKey {
-		accessKey = cred.AccessKey
-	}
-
-	implicitPerm := name == accessKey
-	if !implicitPerm {
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     accessKey,
+		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.GetUserAdminAction,
-			ConditionValues: getConditionValues(r, "", accessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 		IsOwner:         owner,
 		Claims:          claims,
+		DenyOnly:        checkDenyOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
 	}
-	}
 
-	userInfo, err := globalIAMSys.GetUserInfo(name)
+	userInfo, err := globalIAMSys.GetUserInfo(ctx, name)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -216,7 +202,6 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	defer r.Body.Close()
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
@@ -231,23 +216,15 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 	}
 
 	if updReq.IsRemove {
-		err = globalIAMSys.RemoveUsersFromGroup(updReq.Group, updReq.Members)
+		err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
 	} else {
-		err = globalIAMSys.AddUsersToGroup(updReq.Group, updReq.Members)
+		err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
 	}
 
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to load group.
-	for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // GetGroup - /minio/admin/v3/group?group=mygroup1
@@ -290,7 +267,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	groups, err := globalIAMSys.ListGroups()
+	groups, err := globalIAMSys.ListGroups(ctx)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -321,25 +298,18 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 	status := vars["status"]
 
 	var err error
-	if status == statusEnabled {
+	switch status {
-		err = globalIAMSys.SetGroupStatus(group, true)
+	case statusEnabled:
-	} else if status == statusDisabled {
+		err = globalIAMSys.SetGroupStatus(ctx, group, true)
-		err = globalIAMSys.SetGroupStatus(group, false)
+	case statusDisabled:
-	} else {
+		err = globalIAMSys.SetGroupStatus(ctx, group, false)
+	default:
 		err = errInvalidArgument
 	}
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to reload user.
-	for _, nerr := range globalNotificationSys.LoadGroup(group) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
@@ -363,18 +333,10 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
+	if err := globalIAMSys.SetUserStatus(ctx, accessKey, madmin.AccountStatus(status)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to reload user.
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
@@ -405,6 +367,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	userCred, exists := globalIAMSys.GetUser(ctx, accessKey)
+	if exists && (userCred.IsTemp() || userCred.IsServiceAccount()) {
+		// Updating STS credential is not allowed, and this API does not
+		// support updating service accounts.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
 	if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
 		// Incoming access key matches parent user then we should
 		// reject password change requests.
@@ -412,39 +382,21 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	implicitPerm := accessKey == cred.AccessKey
+	checkDenyOnly := false
-	if !implicitPerm {
+	if accessKey == cred.AccessKey {
-		parentUser := cred.ParentUser
+		// Check that there is no explicit deny - otherwise it's allowed
-		if parentUser == "" {
+		// to change one's own password.
-			parentUser = cred.AccessKey
+		checkDenyOnly = true
-		}
-		// For temporary credentials always
-		// the temporary credentials to check
-		// policy without implicit permissions.
-		if cred.IsTemp() && cred.ParentUser == globalActiveCred.AccessKey {
-			parentUser = cred.AccessKey
-		}
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     parentUser,
-			Groups:          cred.Groups,
-			Action:          iampolicy.CreateUserAdminAction,
-			ConditionValues: getConditionValues(r, "", parentUser, claims),
-			IsOwner:         owner,
-			Claims:          claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
 	}
 
-	if implicitPerm && !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
-		AccountName:     accessKey,
+		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.CreateUserAdminAction,
-		ConditionValues: getConditionValues(r, "", accessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 		IsOwner:         owner,
 		Claims:          claims,
-		DenyOnly:        true, // check if changing password is explicitly denied.
+		DenyOnly:        checkDenyOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
@@ -464,25 +416,17 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var uinfo madmin.UserInfo
+	var ureq madmin.AddOrUpdateUserReq
-	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+	if err = json.Unmarshal(configBytes, &ureq); err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
-	if err = globalIAMSys.CreateUser(accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other Minio peers to reload user
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
@@ -522,51 +466,107 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		targetGroups []string
 	)
 
+	// If the request did not set a TargetUser, the service account is
+	// created for the request sender.
 	targetUser = createReq.TargetUser
+	if targetUser == "" {
+		targetUser = cred.AccessKey
+	}
 
-	// Need permission if we are creating a service acccount
+	opts := newServiceAccountOpts{
-	// for a user <> to the request sender
+		accessKey: createReq.AccessKey,
-	if targetUser != "" && targetUser != cred.AccessKey {
+		secretKey: createReq.SecretKey,
+		claims:    make(map[string]interface{}),
+	}
+
+	// Find the user for the request sender (as it may be sent via a service
+	// account or STS account):
+	requestorUser := cred.AccessKey
+	requestorParentUser := cred.AccessKey
+	requestorGroups := cred.Groups
+	requestorIsDerivedCredential := false
+	if cred.IsServiceAccount() || cred.IsTemp() {
+		requestorParentUser = cred.ParentUser
+		requestorIsDerivedCredential = true
+	}
+
+	// Check if we are creating svc account for request sender.
+	isSvcAccForRequestor := false
+	if targetUser == requestorUser || targetUser == requestorParentUser {
+		isSvcAccForRequestor = true
+	}
+
+	// If we are creating svc account for request sender, ensure
+	// that targetUser is a real user (i.e. not derived
+	// credentials).
+	if isSvcAccForRequestor {
+		// Check if adding service account is explicitly denied.
+		//
+		// This allows turning off service accounts for request sender,
+		// if there is no deny statement this call is implicitly enabled.
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     cred.AccessKey,
+			AccountName:     requestorUser,
+			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
 			Claims:          claims,
+			DenyOnly:        true,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
 		}
+
+		if requestorIsDerivedCredential {
+			if requestorParentUser == "" {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
+					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
+				return
+			}
+			targetUser = requestorParentUser
+		}
+		targetGroups = requestorGroups
+
+		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
+		// it is associated with the LDAP/OIDC user properly.
+		for k, v := range cred.Claims {
+			if k == expClaim {
+				continue
+			}
+			opts.claims[k] = v
+		}
+	} else {
+		// Need permission if we are creating a service acccount for a
+		// user <> to the request sender
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     requestorUser,
+			Groups:          requestorGroups,
+			Action:          iampolicy.CreateServiceAccountAdminAction,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			IsOwner:         owner,
+			Claims:          claims,
+			DenyOnly:        true,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
 		}
 
-	var ldapUsername string
+		// In case of LDAP we need to resolve the targetUser to a DN and
-	if globalLDAPConfig.Enabled && targetUser != "" {
+		// query their groups:
-		// If LDAP enabled, service accounts need
+		if globalLDAPConfig.Enabled {
-		// to be created only for LDAP users.
+			opts.claims[ldapUserN] = targetUser // simple username
-		var err error
-		ldapUsername = targetUser
 			targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
-		// targerUser is set to bindDN at this point in time.
+			opts.claims[ldapUser] = targetUser // username DN
-		// targetGroups is set to the groups at this point in time.
-	} else {
-		if cred.IsServiceAccount() || cred.IsTemp() {
-			if cred.ParentUser == "" {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
-					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
-				return
-			}
-			if targetUser == "" {
-				targetUser = cred.ParentUser
-			}
-		}
-		// targetGroups not yet set, so set this to cred.Groups
-		if len(targetGroups) == 0 {
-			targetGroups = cred.Groups
 		}
+
+		// NOTE: if not using LDAP, then internal IDP or open ID is
+		// being used - in the former, group info is enforced when
+		// generated credentials are used to make requests, and in the
+		// latter, a group notion is not supported.
 	}
 
 	var sp *iampolicy.Policy
@@ -578,25 +578,38 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	opts := newServiceAccountOpts{
+	opts.sessionPolicy = sp
-		accessKey:     createReq.AccessKey,
-		secretKey:     createReq.SecretKey,
-		sessionPolicy: sp,
-	}
-	if ldapUsername != "" {
-		opts.ldapUsername = ldapUsername
-	}
 	newCred, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other Minio peers to reload user the service account
+	// Call hook for cluster-replication.
-	for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
+	//
-		if nerr.Err != nil {
+	// FIXME: This wont work in an OpenID situation as the parent credential
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+	// may not be present on peer clusters to provide inherited policies.
-			logger.LogIf(ctx, nerr.Err)
+	// Also, we should not be replicating root user's service account - as
+	// they are not authenticated by a common external IDP, so we skip when
+	// opts.ldapUser == "".
+	if _, isLDAPAccount := opts.claims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Claims:        opts.claims,
+					SessionPolicy: createReq.Policy,
+					Status:        auth.AccountOn,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
 
@@ -703,14 +716,30 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Notify all other Minio peers to reload user the service account
+	// Call site replication hook. Only LDAP accounts are supported for
-	for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
+	// replication operations.
-		if nerr.Err != nil {
+	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+	if err != nil {
-			logger.LogIf(ctx, nerr.Err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Update: &madmin.SRSvcAccUpdate{
+					AccessKey:     accessKey,
+					SecretKey:     opts.secretKey,
+					Status:        opts.status,
+					SessionPolicy: updateReq.NewPolicy,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
-
 	writeSuccessNoContent(w)
 }
 
@@ -828,8 +857,10 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 
 	var targetAccount string
 
+	// If listing is requested for a specific user (who is not the request
+	// sender), check that the user has permissions.
 	user := r.Form.Get("user")
-	if user != "" {
+	if user != "" && user != cred.AccessKey {
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
 			Action:          iampolicy.ListServiceAccountsAdminAction,
@@ -932,16 +963,33 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount)
+	// Save svc acc claims before deletion (for site replication hook).
+	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
+	if err != nil && err != errNoSuchServiceAccount {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	for _, nerr := range globalNotificationSys.DeleteServiceAccount(serviceAccount) {
+	// Call site replication hook. Only LDAP accounts are supported for
-		if nerr.Err != nil {
+	// replication operations.
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
-			logger.LogIf(ctx, nerr.Err)
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Delete: &madmin.SRSvcAccDelete{
+					AccessKey: serviceAccount,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
 
@@ -1006,7 +1054,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	var dataUsageInfo madmin.DataUsageInfo
+	var dataUsageInfo DataUsageInfo
 	var err error
 	if !globalIsGateway {
 		// Load the latest calculated data usage
@@ -1045,19 +1093,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	accountName := cred.AccessKey
-	var policies []string
+	if cred.IsTemp() || cred.IsServiceAccount() {
-	switch globalIAMSys.usersSysType {
+		// For derived credentials, check the parent user's permissions.
-	case MinIOUsersSysType:
+		accountName = cred.ParentUser
-		policies, err = globalIAMSys.PolicyDBGet(accountName, false)
-	case LDAPUsersSysType:
-		parentUser := accountName
-		if cred.ParentUser != "" {
-			parentUser = cred.ParentUser
-		}
-		policies, err = globalIAMSys.PolicyDBGet(parentUser, false, cred.Groups...)
-	default:
-		err = errors.New("should never happen")
 	}
+	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1081,8 +1121,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		if rd || wr {
 			// Fetch the data usage of the current bucket
 			var size uint64
+			var objectsCount uint64
+			var objectsHist map[string]uint64
 			if !dataUsageInfo.LastUpdate.IsZero() {
 				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
+				objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
+				objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
 			}
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
@@ -1093,11 +1137,27 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 					logger.LogIf(ctx, err)
 				}
 			}
+
+			lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
+			quota, _ := globalBucketQuotaSys.Get(bucket.Name)
+			rcfg, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name)
+			tcfg, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
+
 			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
 				Name:                 bucket.Name,
 				Created:              bucket.Created,
 				Size:                 size,
+				Objects:              objectsCount,
+				ObjectSizesHistogram: objectsHist,
 				PrefixUsage:          prefixUsage,
+				Details: &madmin.BucketDetails{
+					Versioning:          globalBucketVersioningSys.Enabled(bucket.Name),
+					VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
+					Replication:         rcfg != nil,
+					Locking:             lcfg.LockEnabled,
+					Quota:               quota,
+					Tagging:             tcfg,
+				},
 				Access: madmin.AccountAccess{
 					Read:  rd,
 					Write: wr,
@@ -1116,6 +1176,16 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 }
 
 // InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
+//
+// Newer API response with policy timestamps is returned with query parameter
+// `v=2` like:
+//
+// GET /minio/admin/v3/info-canned-policy?name={policyName}&v=2
+//
+// The newer API will eventually become the default (and only) one. The older
+// response is to return only the policy JSON. The newer response returns
+// timestamps along with the policy JSON. Both versions are supported for now,
+// for smooth transition to new API.
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
@@ -1126,13 +1196,36 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	name := mux.Vars(r)["name"]
+	policies := newMappedPolicy(name).toSlice()
+	if len(policies) != 1 {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
+		return
+	}
+
+	policyDoc, err := globalIAMSys.InfoPolicy(name)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	buf, err := json.MarshalIndent(policy, "", " ")
+	// Is the new API version being requested?
+	infoPolicyAPIVersion := r.Form.Get("v")
+	if infoPolicyAPIVersion == "2" {
+		buf, err := json.MarshalIndent(policyDoc, "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		w.Write(buf)
+		return
+	} else if infoPolicyAPIVersion != "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("invalid version parameter 'v' supplied")), r.URL)
+		return
+	}
+
+	// Return the older API response value of just the policy json.
+	buf, err := json.MarshalIndent(policyDoc.Policy, "", " ")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1152,7 +1245,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 	}
 
 	bucket := mux.Vars(r)["bucket"]
-	policies, err := globalIAMSys.ListPolicies(bucket)
+	policies, err := globalIAMSys.ListPolicies(ctx, bucket)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1171,9 +1264,6 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
-
 }
 
 // ListCannedPolicies - GET /minio/admin/v3/list-canned-policies
@@ -1187,7 +1277,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	policies, err := globalIAMSys.ListPolicies("")
+	policies, err := globalIAMSys.ListPolicies(ctx, "")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1206,8 +1296,6 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
 }
 
 // RemoveCannedPolicy - DELETE /minio/admin/v3/remove-canned-policy?name=<policy_name>
@@ -1224,17 +1312,19 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
+	if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to delete policy
+	// Call cluster-replication policy creation hook to replicate policy deletion to
-	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
+	// other minio clusters.
-		if nerr.Err != nil {
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+		Type: madmin.SRIAMItemPolicy,
-			logger.LogIf(ctx, nerr.Err)
+		Name: policyName,
-		}
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 }
 
@@ -1264,7 +1354,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	iamPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1276,17 +1372,20 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	if err = globalIAMSys.SetPolicy(policyName, *iamPolicy); err != nil {
+	if err = globalIAMSys.SetPolicy(ctx, policyName, *iamPolicy); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to reload policy
+	// Call cluster-replication policy creation hook to replicate policy to
-	for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
+	// other minio clusters.
-		if nerr.Err != nil {
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+		Type:   madmin.SRIAMItemPolicy,
-			logger.LogIf(ctx, nerr.Err)
+		Name:   policyName,
-		}
+		Policy: iamPolicyBytes,
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 }
 
@@ -1318,16 +1417,20 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		}
 	}
 
-	if err := globalIAMSys.PolicyDBSet(entityName, policyName, isGroup); err != nil {
+	if err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to reload policy
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-	for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
+		Type: madmin.SRIAMItemPolicyMapping,
-		if nerr.Err != nil {
+		PolicyMapping: &madmin.SRPolicyMapping{
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			UserOrGroup: entityName,
-			logger.LogIf(ctx, nerr.Err)
+			IsGroup:     isGroup,
-		}
+			Policy:      policyName,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 }

cmd/admin-handlers.go

@@ -38,7 +38,7 @@ import (
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
@@ -149,9 +149,13 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
 		if nerr.Err != nil {
+			err := AdminError{
+				Code:       AdminUpdateApplyFailure,
+				Message:    nerr.Err.Error(),
+				StatusCode: http.StatusInternalServerError,
+			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
-			err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", nerr.Err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -159,7 +163,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
 	if err != nil {
-		err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", err)
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -186,7 +190,11 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 // ServiceHandler - POST /minio/admin/v3/service?action={action}
 // ----------
-// restarts/stops minio server gracefully. In a distributed setup,
+// Supports following actions:
+// - restart (restarts all the MinIO instances in a setup)
+// - stop (stops all the MinIO instances in a setup)
+// - freeze (freezes all incoming S3 API calls)
+// - unfreeze (unfreezes previously frozen S3 API calls)
 func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Service")
 
@@ -201,6 +209,10 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 		serviceSig = serviceRestart
 	case madmin.ServiceActionStop:
 		serviceSig = serviceStop
+	case madmin.ServiceActionFreeze:
+		serviceSig = serviceFreeze
+	case madmin.ServiceActionUnfreeze:
+		serviceSig = serviceUnFreeze
 	default:
 		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.Application)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
@@ -213,6 +225,8 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
 	case serviceStop:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
+	case serviceFreeze, serviceUnFreeze:
+		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceFreezeAdminAction)
 	}
 	if objectAPI == nil {
 		return
@@ -229,8 +243,15 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	// Reply to the client before restarting, stopping MinIO server.
 	writeSuccessResponseHeadersOnly(w)
 
+	switch serviceSig {
+	case serviceFreeze:
+		freezeServices()
+	case serviceUnFreeze:
+		unfreezeServices()
+	case serviceRestart, serviceStop:
 		globalServiceSignalCh <- serviceSig
 	}
+}
 
 // ServerProperties holds some server information such as, version, region
 // uptime, etc..
@@ -369,10 +390,10 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 		}
 		for k, v := range peerLock.Locks {
 			for _, lockReqInfo := range v {
-				if val, ok := entryMap[lockReqInfo.UID]; ok {
+				if val, ok := entryMap[lockReqInfo.Name]; ok {
 					val.ServerList = append(val.ServerList, peerLock.Addr)
 				} else {
-					entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
+					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 				}
 			}
 		}
@@ -752,13 +773,17 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(200)
 				}
 				// Send whitespace and keep connection open
-				w.Write([]byte(" "))
+				if _, err := w.Write([]byte(" ")); err != nil {
+					return
+				}
 				w.(http.Flusher).Flush()
 			case hr := <-respCh:
 				switch hr.apiErr {
 				case noError:
 					if started {
-						w.Write(hr.respBytes)
+						if _, err := w.Write(hr.respBytes); err != nil {
+							return
+						}
 						w.(http.Flusher).Flush()
 					} else {
 						writeSuccessResponseJSON(w, hr.respBytes)
@@ -783,7 +808,9 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 						w.Header().Set(xhttp.ContentType, string(mimeJSON))
 						w.WriteHeader(hr.apiErr.HTTPStatusCode)
 					}
-					w.Write(errorRespJSON)
+					if _, err := w.Write(errorRespJSON); err != nil {
+						return
+					}
 					w.(http.Flusher).Flush()
 				}
 				break forLoop
@@ -907,12 +934,16 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }
 
+// SpeedtestHandler - reports maximum speed of a cluster by performing PUT and
+// GET operations on the server, supports auto tuning by default by automatically
+// increasing concurrency and stopping when we have reached the limits on the
+// system.
 func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SpeedtestHandler")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -925,6 +956,8 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
+	autotune := r.Form.Get("autotune") == "true"
+	storageClass := r.Form.Get("storage-class")
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -936,21 +969,55 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		concurrent = 32
 	}
 
+	if runtime.GOMAXPROCS(0) < concurrent {
+		concurrent = runtime.GOMAXPROCS(0)
+	}
+
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
 		duration = time.Second * 10
 	}
 
-	results := globalNotificationSys.Speedtest(ctx, size, concurrent, duration)
+	deleteBucket := func() {
+		loc := pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix)
+		objectAPI.DeleteBucket(context.Background(), loc, DeleteBucketOptions{
+			Force:      true,
+			NoRecreate: true,
+		})
+	}
+	defer deleteBucket()
 
-	if err := json.NewEncoder(w).Encode(results); err != nil {
+	// Freeze all incoming S3 API calls before running speedtest.
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+	globalNotificationSys.ServiceFreeze(ctx, true)
+
+	// unfreeze all incoming S3 API calls after speedtest.
+	defer globalNotificationSys.ServiceFreeze(ctx, false)
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	enc := json.NewEncoder(w)
+	ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-keepAliveTicker.C:
+			// Write a blank entry to prevent client from disconnecting
+			if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
 				return
 			}
-
-	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), true)
-
 			w.(http.Flusher).Flush()
+		case result, ok := <-ch:
+			if !ok {
+				return
+			}
+			if err := enc.Encode(result); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		}
+	}
 }
 
 // Admin API errors
@@ -1160,11 +1227,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 				if err := enc.Encode(log); err != nil {
 					return
 				}
-			}
 				if len(logCh) == 0 {
 					// Flush if nothing is queued
 					w.(http.Flusher).Flush()
 				}
+			}
 		case <-keepAliveTicker.C:
 			if len(logCh) > 0 {
 				continue
@@ -1322,6 +1389,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 	ldap := madmin.LDAP{}
 	if globalLDAPConfig.Enabled {
 		ldapConn, err := globalLDAPConfig.Connect()
+		//nolint:gocritic
 		if err != nil {
 			ldap.Status = string(madmin.ItemOffline)
 		} else if ldapConn == nil {
@@ -1403,7 +1471,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 	return madmin.InfoMessage{
 		Mode:         string(mode),
 		Domain:       domain,
-		Region:       globalServerRegion,
+		Region:       globalSite.Region,
 		SQSARN:       globalNotificationSys.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
@@ -1602,8 +1670,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	anonymizeCmdLine := func(cmdLine string) string {
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
-			anonCmdLine := strings.Replace(cmdLine, globalLocalNodeName, "server1", -1)
+			anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
-			return strings.Replace(anonCmdLine, globalMinioConsoleHost, "server1", -1)
+			return strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
 		}
 
 		// Server start command regex groups:
@@ -1799,6 +1867,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 					},
 				})
 			}
+
+			tls := getTLSInfo()
+			isK8s := IsKubernetes()
+			isDocker := IsDocker()
 			healthInfo.Minio.Info = madmin.MinioInfo{
 				Mode:         infoMessage.Mode,
 				Domain:       infoMessage.Domain,
@@ -1811,6 +1883,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				Services:     infoMessage.Services,
 				Backend:      infoMessage.Backend,
 				Servers:      servers,
+				TLS:          &tls,
+				IsKubernetes: &isK8s,
+				IsDocker:     &isDocker,
 			}
 			partialWrite(healthInfo)
 		}
@@ -1825,21 +1900,44 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			if !ok {
 				return
 			}
-			logger.LogIf(ctx, enc.Encode(oinfo))
+			if err := enc.Encode(oinfo); err != nil {
+				return
+			}
+			if len(healthInfoCh) == 0 {
+				// Flush if nothing is queued
 				w.(http.Flusher).Flush()
+			}
 		case <-ticker.C:
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return
 			}
 			w.(http.Flusher).Flush()
 		case <-deadlinedCtx.Done():
-			w.(http.Flusher).Flush()
 			return
 		}
 	}
 
 }
 
+func getTLSInfo() madmin.TLSInfo {
+	tlsInfo := madmin.TLSInfo{
+		TLSEnabled: globalIsTLS,
+		Certs:      []madmin.TLSCert{},
+	}
+
+	if globalIsTLS {
+		for _, c := range globalPublicCerts {
+			tlsInfo.Certs = append(tlsInfo.Certs, madmin.TLSCert{
+				PubKeyAlgo:    c.PublicKeyAlgorithm.String(),
+				SignatureAlgo: c.SignatureAlgorithm.String(),
+				NotBefore:     c.NotBefore,
+				NotAfter:      c.NotAfter,
+			})
+		}
+	}
+	return tlsInfo
+}
+
 // BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
 // ----------
 // Get bandwidth consumption information
@@ -1874,17 +1972,21 @@ func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http
 			}
 		}
 	}()
+
+	enc := json.NewEncoder(w)
 	for {
 		select {
 		case report, ok := <-reportCh:
 			if !ok {
 				return
 			}
-			if err := json.NewEncoder(w).Encode(report); err != nil {
+			if err := enc.Encode(report); err != nil {
-				writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
+			if len(reportCh) == 0 {
+				// Flush if nothing is queued
 				w.(http.Flusher).Flush()
+			}
 		case <-keepAliveTicker.C:
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return
@@ -2027,7 +2129,7 @@ func fetchKMSStatus() madmin.KMS {
 func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit
-	for _, target := range logger.Targets {
+	for _, target := range logger.Targets() {
 		if target.Endpoint() != "" {
 			tgt := target.String()
 			err := checkConnection(target.Endpoint(), 15*time.Second)
@@ -2043,7 +2145,7 @@ func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 		}
 	}
 
-	for _, target := range logger.AuditTargets {
+	for _, target := range logger.AuditTargets() {
 		if target.Endpoint() != "" {
 			tgt := target.String()
 			err := checkConnection(target.Endpoint(), 15*time.Second)
@@ -2096,7 +2198,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
-	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
+	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
 }
 
 // InspectDataHandler - GET /minio/admin/v3/inspect-data
@@ -2129,6 +2231,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+	file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
+
+	// Reject attempts to traverse parent or absolute paths.
+	if strings.Contains(file, "..") || strings.Contains(volume, "..") {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
 
 	var key [32]byte
 	// MUST use crypto/rand
@@ -2166,15 +2275,23 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	zipWriter := zip.NewWriter(encw)
 	defer zipWriter.Close()
 
-	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
+	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, si StatInfo) error {
 		// Prefix host+disk
 		filename = path.Join(host, disk, filename)
+		if si.Dir {
+			filename += "/"
+			si.Size = 0
+		}
+		if si.Mode == 0 {
+			// Not, set it to default.
+			si.Mode = 0600
+		}
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
-			size:    size,
+			size:    si.Size,
-			mode:    0600,
+			mode:    os.FileMode(si.Mode),
-			modTime: modtime,
+			modTime: si.ModTime,
-			isDir:   false,
+			isDir:   si.Dir,
 			sys:     nil,
 		})
 		if zerr != nil {

cmd/admin-handlers_test.go

@@ -28,6 +28,7 @@ import (
 	"net/url"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -40,11 +41,13 @@ type adminErasureTestBed struct {
 	erasureDirs []string
 	objLayer    ObjectLayer
 	router      *mux.Router
+	done        context.CancelFunc
 }
 
 // prepareAdminErasureTestBed - helper function that setups a single-node
 // Erasure backend for admin-handler tests.
 func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, error) {
+	ctx, cancel := context.WithCancel(ctx)
 
 	// reset global variables to start afresh.
 	resetTestGlobals()
@@ -56,11 +59,13 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, erasureDirs, xlErr := initTestErasureObjLayer(ctx)
 	if xlErr != nil {
+		cancel()
 		return nil, xlErr
 	}
 
 	// Initialize minio server config.
 	if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		cancel()
 		return nil, err
 	}
 
@@ -71,9 +76,9 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	newAllSubsystems()
 
-	initAllSubsystems(ctx, objLayer)
+	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.InitStore(objLayer)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -83,12 +88,14 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 		erasureDirs: erasureDirs,
 		objLayer:    objLayer,
 		router:      adminRouter,
+		done:        cancel,
 	}, nil
 }
 
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminErasureTestBed) TearDown() {
+	atb.done()
 	removeRoots(atb.erasureDirs)
 	resetTestGlobals()
 }

cmd/admin-heal-ops.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"sort"
@@ -396,9 +397,6 @@ type healSequence struct {
 	// bucket, and object on which heal seq. was initiated
 	bucket, object string
 
-	// A channel of entities (format, buckets, objects) to heal
-	sourceCh chan healSource
-
 	// A channel of entities with heal result
 	respCh chan healResult
 
@@ -493,7 +491,7 @@ func (h *healSequence) getScannedItemsCount() int64 {
 	defer h.mutex.RUnlock()
 
 	for _, v := range h.scannedItemsMap {
-		count = count + v
+		count += v
 	}
 	return count
 }
@@ -648,11 +646,7 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	h.currentStatus.StartTime = UTCNow()
 	h.mutex.Unlock()
 
-	if h.sourceCh == nil {
 	go h.traverseAndHeal(objAPI)
-	} else {
-		go h.healFromSourceCh()
-	}
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
@@ -696,45 +690,38 @@ func (h *healSequence) logHeal(healType madmin.HealItemType) {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
-	globalHealConfigMu.Lock()
-	opts := globalHealConfig
-	globalHealConfigMu.Unlock()
-
 	// Send heal request
 	task := healTask{
 		bucket:    source.bucket,
 		object:    source.object,
 		versionID: source.versionID,
 		opts:      h.settings,
-		responseCh: h.respCh,
+		respCh:    h.respCh,
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
-	if opts.Bitrot {
+	task.opts.ScanMode = globalHealConfig.ScanMode()
-		task.opts.ScanMode = madmin.HealDeepScan
-	}
-
-	// Wait and proceed if there are active requests
-	waitForLowHTTPReq(opts.IOCount, opts.Sleep)
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++
 	h.lastHealActivity = UTCNow()
 	h.mutex.Unlock()
 
-	globalBackgroundHealRoutine.queueHealTask(task)
+	select {
+	case globalBackgroundHealRoutine.tasks <- task:
+		if serverDebugLog {
+			logger.Info("Task in the queue: %#v", task)
+		}
+	case <-h.ctx.Done():
+		return nil
+	}
 
 	select {
 	case res := <-h.respCh:
 		if !h.reportProgress {
-			// Object might have been deleted, by the time heal
+			if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
-			// was attempted, we should ignore this object and
+				return nil
-			// return the error and not calculate this object
-			// as part of the metrics.
-			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
-				// Return the error so that caller can handle it.
-				return res.err
 			}
 
 			h.mutex.Lock()
@@ -759,11 +746,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		}
 		res.result.Type = healType
 		if res.err != nil {
-			// Object might have been deleted, by the time heal
-			// was attempted, we should ignore this object and return success.
-			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
-				return nil
-			}
 			// Only report object error
 			if healType != madmin.HealItemObject {
 				return res.err
@@ -776,48 +758,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 }
 
-func (h *healSequence) healItemsFromSourceCh() error {
-	for {
-		select {
-		case source, ok := <-h.sourceCh:
-			if !ok {
-				return nil
-			}
-
-			var itemType madmin.HealItemType
-			switch source.bucket {
-			case nopHeal:
-				continue
-			case SlashSeparator:
-				itemType = madmin.HealItemMetadata
-			default:
-				if source.object == "" {
-					itemType = madmin.HealItemBucket
-				} else {
-					itemType = madmin.HealItemObject
-				}
-			}
-
-			if err := h.queueHealTask(source, itemType); err != nil {
-				switch err.(type) {
-				case ObjectExistsAsDirectory:
-				case ObjectNotFound:
-				case VersionNotFound:
-				default:
-					logger.LogIf(h.ctx, fmt.Errorf("Heal attempt failed for %s: %w",
-						pathJoin(source.bucket, source.object), err))
-				}
-			}
-		case <-h.ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (h *healSequence) healFromSourceCh() {
-	h.healItemsFromSourceCh()
-}
-
 func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 	// Start healing the config prefix.
 	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
@@ -862,11 +802,6 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 				object:    object,
 				versionID: versionID,
 			}, madmin.HealItemBucketMetadata)
-			// Object might have been deleted, by the time heal
-			// was attempted we ignore this object an move on.
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				return nil
-			}
 			return err
 		})
 	}
@@ -915,10 +850,8 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly bool) error {
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
-		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 		return err
 	}
-	}
 
 	if bucketsOnly {
 		return nil
@@ -931,9 +864,6 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
-					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-						return nil
-					}
 					return err
 				}
 			}
@@ -943,12 +873,8 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 	}
 
 	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
-		// Object might have been deleted, by the time heal
-		// was attempted we ignore this object an move on.
-		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
-	}
 	return nil
 }
 
@@ -962,6 +888,11 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
+		opts:      &h.settings,
 	}, madmin.HealItemObject)
+
+	// Wait and proceed if there are active requests
+	waitForLowHTTPReq()
+
 	return err
 }

cmd/admin-router.go

@@ -43,8 +43,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 	// Admin router
 	adminRouter := router.PathPrefix(adminPathPrefix).Subrouter()
 
-	/// Service operations
-
 	adminVersions := []string{
 		adminAPIVersionPrefix,
 	}
@@ -71,7 +69,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
 
 		if globalIsDistErasure || globalIsErasure {
-			/// Heal operations
+			// Heal operations
 
 			// Heal processing endpoint.
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
@@ -79,9 +77,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
 
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(gz(httpTraceAll(adminAPI.BackgroundHealStatusHandler)))
-
-			/// Health operations
-
 		}
 
 		// Profiling operations
@@ -106,7 +101,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
 		}
 
-		/// Config import/export bulk operations
+		// Config import/export bulk operations
 		if enableConfigOps {
 			// Get config
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigHandler)))
@@ -191,6 +186,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
+
+			// Tier stats
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
+
+			// Cluster Replication APIs
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
 		}
 
 		if globalIsDistErasure {

cmd/api-datatypes.go

@@ -28,14 +28,10 @@ type DeletedObject struct {
 	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
 	ObjectName            string `xml:"Key,omitempty"`
 	VersionID             string `xml:"VersionId,omitempty"`
-
-	// MinIO extensions to support delete marker replication
-	// Replication status of DeleteMarker
-	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus,omitempty"`
 	// MTime of DeleteMarker on source that needs to be propagated to replica
-	DeleteMarkerMTime DeleteMarkerMTime `xml:"DeleteMarkerMTime,omitempty"`
+	DeleteMarkerMTime DeleteMarkerMTime `xml:"-"`
-	// Status of versioned delete (of object or DeleteMarker)
+	// MinIO extensions to support delete marker replication
-	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus,omitempty"`
+	ReplicationState ReplicationState `xml:"-"`
 }
 
 // DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
@@ -60,8 +56,10 @@ type ObjectToDelete struct {
 	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
 	// Status of versioned delete (of object or DeleteMarker)
 	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus"`
-	// Version ID of delete marker
+	// VersionPurgeStatuses holds the internal
-	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId"`
+	VersionPurgeStatuses string `xml:"VersionPurgeStatuses"`
+	// ReplicateDecisionStr stringified representation of replication decision
+	ReplicateDecisionStr string `xml:"-"`
 }
 
 // createBucketConfiguration container for bucket configuration request from client.

cmd/api-errors.go

@@ -20,9 +20,11 @@ package cmd
 import (
 	"context"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
@@ -80,6 +82,7 @@ const (
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
+	ErrAccessKeyDisabled
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -107,6 +110,7 @@ const (
 	ErrNoSuchBucketPolicy
 	ErrNoSuchBucketLifecycle
 	ErrNoSuchLifecycleConfiguration
+	ErrInvalidLifecycleWithObjectLock
 	ErrNoSuchBucketSSEConfig
 	ErrNoSuchCORSConfiguration
 	ErrNoSuchWebsiteConfiguration
@@ -126,6 +130,7 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
+	ErrReplicationDenyEditError
 	ErrReplicationNoMatchingRuleError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -265,6 +270,16 @@ const (
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
+
+	// Site-Replication errors
+	ErrSiteReplicationInvalidRequest
+	ErrSiteReplicationPeerResp
+	ErrSiteReplicationBackendIssue
+	ErrSiteReplicationServiceAccountError
+	ErrSiteReplicationBucketConfigError
+	ErrSiteReplicationBucketMetaError
+	ErrSiteReplicationIAMError
+
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
 	ErrAdminNoSuchQuotaConfiguration
@@ -383,10 +398,10 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
 	if err != nil {
 		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
-	if globalServerRegion != "" {
+	if globalSite.Region != "" {
 		switch errCode {
 		case ErrAuthorizationHeaderMalformed:
-			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
+			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 			return apiErr
 		}
 	}
@@ -442,7 +457,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -500,6 +515,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrAccessKeyDisabled: {
+		Code:           "InvalidAccessKeyId",
+		Description:    "Your account is disabled; please contact your administrator.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -565,6 +585,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The lifecycle configuration does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrInvalidLifecycleWithObjectLock: {
+		Code:           "InvalidLifecycleWithObjectLock",
+		Description:    "The lifecycle configuration containing MaxNoncurrentVersions is not supported with object locking",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchBucketSSEConfig: {
 		Code:           "ServerSideEncryptionConfigurationNotFoundError",
 		Description:    "The server side encryption configuration was not found",
@@ -662,7 +687,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
-		Description:    "All access to this bucket has been disabled.",
+		Description:    "All access to this resource has been disabled.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedPolicy: {
@@ -870,6 +895,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching replication rule found for this object prefix",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationDenyEditError: {
+		Code:           "XMinioReplicationDenyEdit",
+		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
@@ -961,7 +991,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusNotFound,
 	},
 
-	/// Bucket notification related errors.
+	// Bucket notification related errors.
 	ErrEventNotification: {
 		Code:           "InvalidArgument",
 		Description:    "A specified event is not supported for notifications.",
@@ -1108,14 +1138,14 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusForbidden,
 	},
 
-	/// S3 extensions.
+	// S3 extensions.
 	ErrContentSHA256Mismatch: {
 		Code:           "XAmzContentSHA256Mismatch",
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 
-	/// MinIO extensions.
+	// MinIO extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
@@ -1267,6 +1297,43 @@ var errorCodes = errorCodeMap{
 		Description:    errObjectTampered.Error(),
 		HTTPStatusCode: http.StatusPartialContent,
 	},
+
+	ErrSiteReplicationInvalidRequest: {
+		Code:           "XMinioSiteReplicationInvalidRequest",
+		Description:    "Invalid site-replication request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSiteReplicationPeerResp: {
+		Code:           "XMinioSiteReplicationPeerResp",
+		Description:    "Error received when contacting a peer site",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBackendIssue: {
+		Code:           "XMinioSiteReplicationBackendIssue",
+		Description:    "Error when requesting object layer backend",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationServiceAccountError: {
+		Code:           "XMinioSiteReplicationServiceAccountError",
+		Description:    "Site replication related service account error",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBucketConfigError: {
+		Code:           "XMinioSiteReplicationBucketConfigError",
+		Description:    "Error while configuring replication on a bucket",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBucketMetaError: {
+		Code:           "XMinioSiteReplicationBucketMetaError",
+		Description:    "Error while replicating bucket metadata",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationIAMError: {
+		Code:           "XMinioSiteReplicationIAMError",
+		Description:    "Error while replicating an IAM item",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
@@ -1313,8 +1380,8 @@ var errorCodes = errorCodeMap{
 	},
 	ErrBackendDown: {
 		Code:           "XMinioBackendDown",
-		Description:    "Object storage backend is unreachable",
+		Description:    "Remote backend is unreachable",
-		HTTPStatusCode: http.StatusServiceUnavailable,
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrIncorrectContinuationToken: {
 		Code:           "InvalidArgument",
@@ -1785,12 +1852,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrOperationTimedOut
-	select {
+	if contextCanceled(ctx) {
-	case <-ctx.Done():
 		if ctx.Err() == context.Canceled {
 			return ErrClientDisconnected
 		}
-	default:
 	}
 
 	switch err {
@@ -1909,6 +1974,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchKey
 	case MethodNotAllowed:
 		apiErr = ErrMethodNotAllowed
+	case ObjectLocked:
+		apiErr = ErrObjectLocked
 	case InvalidVersionID:
 		apiErr = ErrInvalidVersionID
 	case VersionNotFound:
@@ -1965,8 +2032,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrReplicationConfigurationNotFoundError
 	case BucketRemoteDestinationNotFound:
 		apiErr = ErrRemoteDestinationNotFoundError
-	case BucketReplicationDestinationMissingLock:
-		apiErr = ErrReplicationDestinationMissingLock
 	case BucketRemoteTargetNotFound:
 		apiErr = ErrRemoteTargetNotFoundError
 	case BucketRemoteConnectionErr:
@@ -2027,6 +2092,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	default:
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648
+		//nolint:gocritic
 		if _, ferr := fmt.Fscanf(strings.NewReader(err.Error()),
 			"request declared a Content-Length of %d but only wrote %d bytes",
 			&ie, &iw); ferr != nil {
@@ -2082,6 +2148,11 @@ func toAPIError(ctx context.Context, err error) APIError {
 		}
 	}
 
+	if apiErr.Code == "XMinioBackendDown" {
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
+		return apiErr
+	}
+
 	if apiErr.Code == "InternalError" {
 		// If we see an internal error try to interpret
 		// any underlying errors if possible depending on
@@ -2177,6 +2248,26 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 			// Add more Gateway SDKs here if any in future.
 		default:
+			//nolint:gocritic
+			if errors.Is(err, errMalformedEncoding) {
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			} else if errors.Is(err, errChunkTooBig) {
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			} else if errors.Is(err, strconv.ErrRange) {
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			} else {
 				apiErr = APIError{
 					Code:           apiErr.Code,
 					Description:    fmt.Sprintf("%s: cause(%v)", apiErr.Description, err),
@@ -2184,6 +2275,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 				}
 			}
 		}
+	}
 
 	return apiErr
 }
@@ -2206,7 +2298,7 @@ func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		Resource:   resource,
-		Region:     globalServerRegion,
+		Region:     globalSite.Region,
 		RequestID:  requestID,
 		HostID:     hostID,
 	}

cmd/api-headers.go

@@ -52,7 +52,7 @@ func setCommonHeaders(w http.ResponseWriter) {
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
-	if region := globalServerRegion; region != "" {
+	if region := globalSite.Region; region != "" {
 		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
 	w.Header().Set(xhttp.AcceptRanges, "bytes")

cmd/api-response.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -38,7 +39,7 @@ const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
 	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
 	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
+	maxDeleteList     = 1000                       // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
 )
@@ -262,7 +263,7 @@ func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error
 	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
-// StringMap is a map[string]string.
+// StringMap is a map[string]string
 type StringMap map[string]string
 
 // MarshalXML - StringMap marshals into XML.
@@ -423,10 +424,10 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	}
 
 	for _, bucket := range buckets {
-		var listbucket = Bucket{}
+		listbuckets = append(listbuckets, Bucket{
-		listbucket.Name = bucket.Name
+			Name:         bucket.Name,
-		listbucket.CreationDate = bucket.Created.UTC().Format(iso8601TimeFormat)
+			CreationDate: bucket.Created.UTC().Format(iso8601TimeFormat),
-		listbuckets = append(listbuckets, listbucket)
+		})
 	}
 
 	data.Owner = owner
@@ -570,6 +571,14 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		content.Owner = owner
 		if metadata {
 			content.UserMetadata = make(StringMap)
+			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+			case crypto.S3:
+				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+			case crypto.S3KMS:
+				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+			case crypto.SSEC:
+				content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+			}
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata
@@ -715,9 +724,6 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, err
 	if !quiet {
 		deleteResp.DeletedObjects = deletedObjects
 	}
-	if len(errs) == len(deletedObjects) {
-		deleteResp.DeletedObjects = nil
-	}
 	deleteResp.Errors = errs
 	return deleteResp
 }
@@ -731,7 +737,6 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	w.WriteHeader(statusCode)
 	if response != nil {
 		w.Write(response)
-		w.(http.Flusher).Flush()
 	}
 }
 
@@ -782,9 +787,9 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
 	case "InvalidRegion":
-		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalServerRegion)
+		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region)
 	case "AuthorizationHeaderMalformed":
-		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
+		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 	}
 
 	// Generate error response.

cmd/api-router.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/gzhttp"
+	"github.com/minio/console/restapi"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/wildcard"
@@ -42,6 +43,18 @@ func setHTTPServer(h *xhttp.Server) {
 	globalObjLayerMutex.Unlock()
 }
 
+func newConsoleServerFn() *restapi.Server {
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
+	return globalConsoleSrv
+}
+
+func setConsoleSrv(srv *restapi.Server) {
+	globalObjLayerMutex.Lock()
+	globalConsoleSrv = srv
+	globalObjLayerMutex.Unlock()
+}
+
 func newObjectLayerFn() ObjectLayer {
 	globalObjLayerMutex.RLock()
 	defer globalObjLayerMutex.RUnlock()
@@ -301,7 +314,8 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("restoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "")
 
-		/// Bucket operations
+		// Bucket operations
+
 		// GetBucketLocation
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "")
@@ -452,7 +466,7 @@ func registerAPIRouter(router *mux.Router) {
 			collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler)))))
 	}
 
-	/// Root operation
+	// Root operation
 
 	// ListenNotification
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(

cmd/auth-handler.go

@@ -138,7 +138,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	var owner bool
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
-		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		getRequestAuthType(r) == authTypeSigned {
 		// We only support admin credentials to access admin APIs.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
@@ -238,19 +238,6 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
 
-	// If LDAP claim key is set, return here.
-	if _, ok := claims.MapClaims[ldapUser]; ok {
-		return claims.Map(), nil
-	}
-
-	// Session token must have a policy, reject requests without policy
-	// claim.
-	_, pokOpenID := claims.MapClaims[iamPolicyClaimNameOpenID()]
-	_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
-	if !pokOpenID && !pokSA {
-		return nil, errAuthentication
-	}
-
 	return claims.Map(), nil
 }
 
@@ -299,7 +286,7 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
@@ -487,6 +474,26 @@ func setAuthHandler(h http.Handler) http.Handler {
 	// handler for validating incoming authorization headers.
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		aType := getRequestAuthType(r)
+		if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
+			// Verify if date headers are set, if not reject the request
+			amzDate, errCode := parseAmzDateHeader(r)
+			if errCode != ErrNone {
+				// All our internal APIs are sensitive towards Date
+				// header, for all requests where Date header is not
+				// present we will reject such clients.
+				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
+				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
+				return
+			}
+			// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
+			// or in the future, reject request otherwise.
+			curTime := UTCNow()
+			if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
+				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
+				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
+				return
+			}
+		}
 		if isSupportedS3AuthType(aType) || aType == authTypeJWT || aType == authTypeSTS {
 			h.ServeHTTP(w, r)
 			return
@@ -509,7 +516,7 @@ func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool,
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypePresigned, authTypeSigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
 			return cred, owner, s3Err
 		}
@@ -576,7 +583,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {

cmd/auth-handler_test.go

@@ -364,9 +364,12 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	newAllSubsystems()
 
-	initAllSubsystems(context.Background(), objLayer)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
-	globalIAMSys.InitStore(objLayer)
+	initConfigSubsystem(ctx, objLayer)
+
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -392,10 +395,9 @@ func TestIsReqAuthenticated(t *testing.T) {
 		{mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
-	ctx := context.Background()
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerRegion, serviceS3)
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
 		if s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
@@ -433,15 +435,15 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}
 }
 
 func TestValidateAdminSignature(t *testing.T) {
-
+	ctx, cancel := context.WithCancel(context.Background())
-	ctx := context.Background()
+	defer cancel()
 
 	objLayer, fsDir, err := prepareFS()
 	if err != nil {
@@ -455,9 +457,9 @@ func TestValidateAdminSignature(t *testing.T) {
 
 	newAllSubsystems()
 
-	initAllSubsystems(context.Background(), objLayer)
+	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.InitStore(objLayer)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {

cmd/background-heal-ops.go

@@ -19,10 +19,9 @@ package cmd
 
 import (
 	"context"
-	"time"
+	"runtime"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/logger"
 )
 
 // healTask represents what to heal along with options
@@ -35,7 +34,7 @@ type healTask struct {
 	versionID string
 	opts      madmin.HealOpts
 	// Healing response will be sent here
-	responseCh chan healResult
+	respCh chan healResult
 }
 
 // healResult represents a healing result with a possible error
@@ -47,53 +46,36 @@ type healResult struct {
 // healRoutine receives heal tasks, to heal buckets, objects and format.json
 type healRoutine struct {
 	tasks   chan healTask
-	doneCh chan struct{}
+	workers int
 }
 
-// Add a new task in the tasks queue
+func systemIO() int {
-func (h *healRoutine) queueHealTask(task healTask) {
-	h.tasks <- task
-}
-
-func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
-	// No need to wait run at full speed.
-	if maxIO <= 0 {
-		return
-	}
-
-	// At max 10 attempts to wait with 100 millisecond interval before proceeding
-	waitTick := 100 * time.Millisecond
-
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	maxIOFn := func() int {
+	return int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
-		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
 }
 
-	tmpMaxWait := maxWait
+func waitForLowHTTPReq() {
+	var currentIO func() int
 	if httpServer := newHTTPServerFn(); httpServer != nil {
-		// Any requests in progress, delay the heal.
+		currentIO = httpServer.GetRequestCount
-		for httpServer.GetRequestCount() >= maxIOFn() {
-			if tmpMaxWait > 0 {
-				if tmpMaxWait < waitTick {
-					time.Sleep(tmpMaxWait)
-				} else {
-					time.Sleep(waitTick)
-				}
-				tmpMaxWait = tmpMaxWait - waitTick
-			}
-			if tmpMaxWait <= 0 {
-				if intDataUpdateTracker.debug {
-					logger.Info("waitForLowHTTPReq: waited max %s, resuming", maxWait)
-				}
-				break
 	}
+
+	globalHealConfig.Wait(currentIO, systemIO)
 }
+
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	// Run the background healer
+	globalBackgroundHealRoutine = newHealRoutine()
+	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
+		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
 	}
+
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
 }
 
 // Wait for heal requests and process them
-func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
+func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 	for {
 		select {
 		case task, ok := <-h.tasks:
@@ -105,6 +87,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			var err error
 			switch task.bucket {
 			case nopHeal:
+				task.respCh <- healResult{err: errSkipFile}
 				continue
 			case SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
@@ -115,10 +98,8 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 					res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
 				}
 			}
-			task.responseCh <- healResult{result: res, err: err}
 
-		case <-h.doneCh:
+			task.respCh <- healResult{result: res, err: err}
-			return
 		case <-ctx.Done():
 			return
 		}
@@ -126,9 +107,13 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func newHealRoutine() *healRoutine {
+	workers := runtime.GOMAXPROCS(0) / 2
+	if workers == 0 {
+		workers = 4
+	}
 	return &healRoutine{
 		tasks:   make(chan healTask),
-		doneCh: make(chan struct{}),
+		workers: workers,
 	}
 
 }

cmd/background-newdisks-heal-ops.go

@@ -18,12 +18,12 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"sort"
 	"strings"
 	"sync"
@@ -308,14 +308,6 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 
 }
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
-	go globalBackgroundHealRoutine.run(ctx, objAPI)
-
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
-}
-
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
@@ -337,10 +329,10 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
 			if len(healDisks) > 0 {
 				// Reformat disks
-				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
+				bgSeq.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
 
 				// Ensure that reformatting disks is finished
-				bgSeq.sourceCh <- healSource{bucket: nopHeal}
+				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)
 
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
@@ -383,15 +375,13 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 
 			buckets, _ := z.ListBuckets(ctx)
 
-			buckets = append(buckets, BucketInfo{
-				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			})
-
 			// Buckets data are dispersed in multiple zones/sets, make
 			// sure to heal all bucket metadata configuration.
-			buckets = append(buckets, []BucketInfo{
+			buckets = append(buckets, BucketInfo{
-				{Name: pathJoin(minioMetaBucket, bucketMetaPrefix)},
+				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			}...)
+			}, BucketInfo{
+				Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+			})
 
 			// Heal latest buckets first.
 			sort.Slice(buckets, func(i, j int) bool {
@@ -420,7 +410,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 							// So someone changed the drives underneath, healing tracker missing.
 							tracker, err := loadHealingTracker(ctx, disk)
 							if err != nil {
-								logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
+								logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool",
+									disk, humanize.Ordinal(i+1))
 								tracker = newHealingTracker(disk)
 							}
 
@@ -442,17 +433,18 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								return
 							}
 
-							err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, buckets, tracker)
+							err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, tracker.QueuedBuckets, tracker)
 							if err != nil {
 								logger.LogIf(ctx, err)
 								continue
 							}
 
-							logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
+							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
-							var buf bytes.Buffer
+								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
-							tracker.printTo(&buf)
+							logger.Info("Summary:\n")
-							logger.Info("Summary:\n%s", buf.String())
+							tracker.printTo(os.Stdout)
 							logger.LogIf(ctx, tracker.delete(ctx))
+							logger.Info("\n")
 
 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())

cmd/bitrot-streaming.go

@@ -57,6 +57,10 @@ func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
 		b.closeWithErr(err)
 		return n, err
 	}
+	if n != len(p) {
+		err = io.ErrShortWrite
+		b.closeWithErr(err)
+	}
 	return n, err
 }
 

cmd/bitrot.go

@@ -27,8 +27,10 @@ import (
 	"io"
 
 	"github.com/minio/highwayhash"
-	"github.com/minio/minio/internal/logger"
 	"golang.org/x/crypto/blake2b"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/logger"
 )
 
 // magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
@@ -172,8 +174,8 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 		return errFileCorrupt
 	}
 
-	bufp := xlPoolSmall.Get().(*[]byte)
+	bufp := xioutil.ODirectPoolSmall.Get().(*[]byte)
-	defer xlPoolSmall.Put(bufp)
+	defer xioutil.ODirectPoolSmall.Put(bufp)
 
 	for left > 0 {
 		// Read expected hash...

cmd/bootstrap-peer-server.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"reflect"
 	"runtime"
 	"time"
 
@@ -32,6 +33,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -52,8 +54,8 @@ type bootstrapRESTServer struct{}
 // ServerSystemConfig - captures information about server configuration.
 type ServerSystemConfig struct {
 	MinioPlatform  string
-	MinioRuntime   string
 	MinioEndpoints EndpointServerPools
+	MinioEnv       map[string]string
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -82,15 +84,49 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 					s2.MinioEndpoints[i].Endpoints[j])
 			}
 		}
-
+	}
+	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
+		var missing []string
+		var mismatching []string
+		for k, v := range s1.MinioEnv {
+			ev, ok := s2.MinioEnv[k]
+			if !ok {
+				missing = append(missing, k)
+			} else if v != ev {
+				mismatching = append(mismatching, k)
+			}
+		}
+		if len(mismatching) > 0 {
+			return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
+		}
+		return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
 	}
 	return nil
 }
 
+var skipEnvs = map[string]struct{}{
+	"MINIO_OPTS":         {},
+	"MINIO_CERT_PASSWD":  {},
+	"MINIO_SERVER_DEBUG": {},
+}
+
 func getServerSystemCfg() ServerSystemConfig {
+	envs := env.List("MINIO_")
+	envValues := make(map[string]string, len(envs))
+	for _, envK := range envs {
+		// skip certain environment variables as part
+		// of the whitelist and could be configured
+		// differently on each nodes, update skipEnvs()
+		// map if there are such environment values
+		if _, ok := skipEnvs[envK]; ok {
+			continue
+		}
+		envValues[envK] = env.Get(envK, "")
+	}
 	return ServerSystemConfig{
 		MinioPlatform:  fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
 		MinioEndpoints: globalEndpoints,
+		MinioEnv:       envValues,
 	}
 }
 
@@ -101,7 +137,6 @@ func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Reque
 	ctx := newContext(r, w, "VerifyHandler")
 	cfg := getServerSystemCfg()
 	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
-	w.(http.Flusher).Flush()
 }
 
 // registerBootstrapRESTHandlers - register bootstrap rest router.
@@ -226,7 +261,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		Path:   bootstrapRESTPath,
 	}
 
-	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
 	restClient.HealthCheckFn = nil
 
 	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}

cmd/bucket-encryption-handlers.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -95,6 +97,20 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	writeSuccessResponseHeadersOnly(w)
 }
 

cmd/bucket-encryption.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"io"
 
-	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 )
 
 // BucketSSEConfigSys - in-memory cache of bucket encryption config
@@ -33,7 +33,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 }
 
 // Get - gets bucket encryption config for the given bucket.
-func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
+func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerFn()
 		if objAPI == nil {
@@ -47,8 +47,8 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, e
 }
 
 // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.
-func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
+func validateBucketSSEConfig(r io.Reader) (*sse.BucketSSEConfig, error) {
-	encConfig, err := bucketsse.ParseBucketSSEConfig(r)
+	encConfig, err := sse.ParseBucketSSEConfig(r)
 	if err != nil {
 		return nil, err
 	}

cmd/bucket-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
@@ -37,8 +38,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
@@ -130,8 +133,6 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	// Add/update buckets that are not registered with the DNS
 	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
 	g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice)).WithConcurrency(50)
-	ctx, cancel := g.WithCancelOnError(GlobalContext)
-	defer cancel()
 
 	for index := range bucketsToBeUpdatedSlice {
 		index := index
@@ -140,10 +141,13 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		}, index)
 	}
 
-	if err := g.WaitErr(); err != nil {
+	ctx := GlobalContext
+	for _, err := range g.Wait() {
+		if err != nil {
 			logger.LogIf(ctx, err)
 			return
 		}
+	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
 		logger.LogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
@@ -207,7 +211,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := globalServerRegion
+	region := globalSite.Region
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,
@@ -411,16 +415,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	const maxBodySize = 2 * 100000 * 1024
 
 	// Unmarshal list of keys to be deleted.
-	deleteObjects := &DeleteObjectsRequest{}
+	deleteObjectsReq := &DeleteObjectsRequest{}
-	if err := xmlDecoder(r.Body, deleteObjects, maxBodySize); err != nil {
+	if err := xmlDecoder(r.Body, deleteObjectsReq, maxBodySize); err != nil {
 		logger.LogIf(ctx, err, logger.Application)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Convert object name delete objects if it has `/` in the beginning.
-	for i := range deleteObjects.Objects {
+	for i := range deleteObjectsReq.Objects {
-		deleteObjects.Objects[i].ObjectName = trimLeadingSlash(deleteObjects.Objects[i].ObjectName)
+		deleteObjectsReq.Objects[i].ObjectName = trimLeadingSlash(deleteObjectsReq.Objects[i].ObjectName)
 	}
 
 	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
@@ -439,8 +443,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
-	// Return Malformed XML as S3 spec if the list of objects is empty
+	// Return Malformed XML as S3 spec if the number of objects is empty
-	if len(deleteObjects.Objects) == 0 {
+	if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
 		return
 	}
@@ -452,11 +456,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	var (
-		hasLockEnabled, replicateSync bool
+		hasLockEnabled bool
+		dsc            ReplicateDecision
 		goi            ObjectInfo
 		gerr           error
 	)
-	replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects)
+	replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjectsReq.Objects)
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
 		hasLockEnabled = true
 	}
@@ -464,16 +469,23 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	versioned := globalBucketVersioningSys.Enabled(bucket)
 	suspended := globalBucketVersioningSys.Suspended(bucket)
 
-	dErrs := make([]DeleteError, len(deleteObjects.Objects))
+	type deleteResult struct {
-	oss := make([]*objSweeper, len(deleteObjects.Objects))
+		delInfo DeletedObject
-	for index, object := range deleteObjects.Objects {
+		errInfo DeleteError
+	}
+
+	deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
+
+	oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
+
+	for index, object := range deleteObjectsReq.Objects {
 		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
 			if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
 				return
 			}
 			apiErr := errorCodes.ToAPIErr(apiErrCode)
-			dErrs[index] = DeleteError{
+			deleteResults[index].errInfo = DeleteError{
 				Code:      apiErr.Code,
 				Message:   apiErr.Description,
 				Key:       object.ObjectName,
@@ -485,7 +497,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			if _, err := uuid.Parse(object.VersionID); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("invalid version-id specified %w", err))
 				apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion)
-				dErrs[index] = DeleteError{
+				deleteResults[index].errInfo = DeleteError{
 					Code:      apiErr.Code,
 					Message:   apiErr.Description,
 					Key:       object.ObjectName,
@@ -514,30 +526,24 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if replicateDeletes {
-			replicate, repsync := checkReplicateDelete(ctx, bucket, ObjectToDelete{
+			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectName: object.ObjectName,
 				VersionID:  object.VersionID,
-			}, goi, gerr)
+			}, goi, opts, gerr)
-			replicateSync = repsync
+			if dsc.ReplicateAny() {
-			if replicate {
-				if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object.ObjectName); apiErrCode != ErrNone {
-					if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
-						writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
-						return
-					}
-					continue
-				}
 				if object.VersionID != "" {
 					object.VersionPurgeStatus = Pending
+					object.VersionPurgeStatuses = dsc.PendingStatus()
 				} else {
-					object.DeleteMarkerReplicationStatus = string(replication.Pending)
+					object.DeleteMarkerReplicationStatus = dsc.PendingStatus()
 				}
+				object.ReplicateDecisionStr = dsc.String()
 			}
 		}
 		if object.VersionID != "" && hasLockEnabled {
 			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
 				apiErr := errorCodes.ToAPIErr(apiErrCode)
-				dErrs[index] = DeleteError{
+				deleteResults[index].errInfo = DeleteError{
 					Code:      apiErr.Code,
 					Message:   apiErr.Description,
 					Key:       object.ObjectName,
@@ -568,7 +574,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		Versioned:        versioned,
 		VersionSuspended: suspended,
 	})
-	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
+
 	for i := range errs {
 		// DeleteMarkerVersionID is not used specifically to avoid
 		// lookup errors, since DeleteMarkerVersionID is only
@@ -577,20 +583,21 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		objToDel := ObjectToDelete{
 			ObjectName:                    dObjects[i].ObjectName,
 			VersionID:                     dObjects[i].VersionID,
-			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus,
+			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus(),
-			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
+			VersionPurgeStatuses:          dObjects[i].ReplicationState.VersionPurgeStatusInternal,
+			DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal,
+			ReplicateDecisionStr:          dObjects[i].ReplicationState.ReplicateDecisionStr,
 		}
 		dindex := objectsToDelete[objToDel]
 		if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
 			if replicateDeletes {
-				dObjects[i].DeleteMarkerReplicationStatus = deleteList[i].DeleteMarkerReplicationStatus
+				dObjects[i].ReplicationState = deleteList[i].ReplicationState()
-				dObjects[i].VersionPurgeStatus = deleteList[i].VersionPurgeStatus
 			}
-			deletedObjects[dindex] = dObjects[i]
+			deleteResults[dindex].delInfo = dObjects[i]
 			continue
 		}
 		apiErr := toAPIError(ctx, errs[i])
-		dErrs[dindex] = DeleteError{
+		deleteResults[dindex].errInfo = DeleteError{
 			Code:      apiErr.Code,
 			Message:   apiErr.Description,
 			Key:       deleteList[i].ObjectName,
@@ -598,15 +605,18 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 	}
 
-	var deleteErrors []DeleteError
+	// Generate response
-	for _, dErr := range dErrs {
+	var deleteErrors = make([]DeleteError, 0, len(deleteObjectsReq.Objects))
-		if dErr.Code != "" {
+	var deletedObjects = make([]DeletedObject, 0, len(deleteObjectsReq.Objects))
-			deleteErrors = append(deleteErrors, dErr)
+	for _, deleteResult := range deleteResults {
+		if deleteResult.errInfo.Code != "" {
+			deleteErrors = append(deleteErrors, deleteResult.errInfo)
+		} else {
+			deletedObjects = append(deletedObjects, deleteResult.delInfo)
 		}
 	}
 
-	// Generate response
+	response := generateMultiDeleteResponse(deleteObjectsReq.Quiet, deletedObjects, deleteErrors)
-	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// Write success response.
@@ -617,12 +627,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if replicateDeletes {
-			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
+			if dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending {
 				dv := DeletedObjectReplicationInfo{
 					DeletedObject: dobj,
 					Bucket:        bucket,
 				}
-				scheduleReplicationDelete(ctx, dv, objectAPI, replicateSync)
+				scheduleReplicationDelete(ctx, dv, objectAPI)
 			}
 		}
 
@@ -730,7 +740,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(ctx, bucket, false)
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -773,6 +783,17 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
+	if _, ok := err.(BucketExists); ok {
+		// Though bucket exists locally, we send the site-replication
+		// hook to ensure all sites have this bucket. If the hook
+		// succeeds, the client will still receive a bucket exists
+		// message.
+		err2 := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+		if err2 != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -781,6 +802,13 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	// Load updated bucket metadata into memory.
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
+	// Call site replication hook
+	err = globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Make sure to add Location information here only for bucket
 	if cp := pathClean(r.URL.Path); cp != "" {
 		w.Header().Set(xhttp.Location, cp) // Clean any trailing slashes.
@@ -885,7 +913,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
 		// S3 feature to replace ${filename} found in Key form field
 		// by the filename attribute passed in multipart
-		formValues.Set("Key", strings.Replace(formValues.Get("Key"), "${filename}", fileName, -1))
+		formValues.Set("Key", strings.ReplaceAll(formValues.Get("Key"), "${filename}", fileName))
 	}
 	object := trimLeadingSlash(formValues.Get("Key"))
 
@@ -980,7 +1008,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	// get gateway encryption options
 	var opts ObjectOptions
@@ -1179,7 +1210,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	writeSuccessResponseHeadersOnly(w)
+	writeResponse(w, http.StatusOK, nil, mimeXML)
 }
 
 // DeleteBucketHandler - Delete bucket
@@ -1240,7 +1271,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, forceDelete); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1249,7 +1280,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 		if globalDNSConfig != nil {
 			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, pl1ease fix it manually", err2))
+				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
 			}
 		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
@@ -1258,6 +1289,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 
+	// Call site replication hook.
+	if err := globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessNoContent(w)
 
@@ -1323,6 +1360,20 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
+		Bucket:           bucket,
+		ObjectLockConfig: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1384,6 +1435,12 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketTaggingAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -1408,6 +1465,20 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeTags,
+		Bucket: bucket,
+		Tags:   &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1476,6 +1547,14 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeTags,
+		Bucket: bucket,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1507,7 +1586,10 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return
@@ -1519,9 +1601,9 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	sameTarget, err := validateReplicationDestination(ctx, bucket, replicationConfig)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig)
-	if err != nil {
+	if apiErr != noError {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 	// Validate the received bucket replication config
@@ -1609,6 +1691,10 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if err := globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1646,32 +1732,19 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	var usageInfo BucketUsageInfo
-	bucketStats := globalNotificationSys.GetClusterBucketStats(r.Context(), bucket)
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	bucketReplStats := BucketReplicationStats{}
+	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
-	// sum up metrics from each node in the cluster
+		usageInfo = dataUsageInfo.BucketsUsage[bucket]
-	for _, bucketStat := range bucketStats {
-		bucketReplStats.FailedCount += bucketStat.ReplicationStats.FailedCount
-		bucketReplStats.FailedSize += bucketStat.ReplicationStats.FailedSize
-		bucketReplStats.PendingCount += bucketStat.ReplicationStats.PendingCount
-		bucketReplStats.PendingSize += bucketStat.ReplicationStats.PendingSize
-		bucketReplStats.ReplicaSize += bucketStat.ReplicationStats.ReplicaSize
-		bucketReplStats.ReplicatedSize += bucketStat.ReplicationStats.ReplicatedSize
 	}
-	// add initial usage from the time of cluster up
-	usageStat := globalReplicationStats.GetInitialUsage(bucket)
-	bucketReplStats.FailedCount += usageStat.FailedCount
-	bucketReplStats.FailedSize += usageStat.FailedSize
-	bucketReplStats.PendingCount += usageStat.PendingCount
-	bucketReplStats.PendingSize += usageStat.PendingSize
-	bucketReplStats.ReplicaSize += usageStat.ReplicaSize
-	bucketReplStats.ReplicatedSize += usageStat.ReplicatedSize
 
-	if err := json.NewEncoder(w).Encode(&bucketReplStats); err != nil {
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+
+	enc := json.NewEncoder(w)
+	if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	w.(http.Flusher).Flush()
 }
 
 // ResetBucketReplicationStateHandler - starts a replication reset for all objects in a bucket which
@@ -1680,12 +1753,16 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 // remote target is entirely lost,and previously replicated objects need to be re-synced.
 func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ResetBucketReplicationState")
-
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	durationStr := r.Form.Get("older-than")
+	durationStr := r.URL.Query().Get("older-than")
+	arn := r.URL.Query().Get("arn")
+	resetID := r.URL.Query().Get("reset-id")
+	if resetID == "" {
+		resetID = mustGetUUID()
+	}
 	var (
 		days time.Duration
 		err  error
@@ -1726,9 +1803,31 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoMatchingRuleError), r.URL)
 		return
 	}
-	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, config.RoleArn)
+	tgtArns := config.FilterTargetArns(
+		replication.ObjectOpts{
+			OpType:    replication.ResyncReplicationType,
+			TargetArn: arn})
+
+	if len(tgtArns) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("Remote target ARN %s missing/not eligible for replication resync", arn),
+		}), r.URL)
+		return
+	}
+
+	if len(tgtArns) > 1 && arn == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("ARN should be specified for replication reset"),
+		}), r.URL)
+		return
+	}
+	var rinfo ResyncTargetsInfo
+	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0])
 	target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
-	target.ResetID = mustGetUUID()
+	target.ResetID = resetID
+	rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
 		switch err.(type) {
 		case BucketRemoteConnectionErr:
@@ -1752,7 +1851,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	data, err := json.Marshal(target.ResetID)
+	data, err := json.Marshal(rinfo)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/bucket-handlers_test.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -656,7 +657,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 
 // Wrapper for calling DeleteMultipleObjects HTTP handler tests for both Erasure multiple disks and single node setup.
 func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testAPIDeleteMultipleObjectsHandler, []string{"DeleteMultipleObjects"})
+	ExecObjectLayerAPITest(t, testAPIDeleteMultipleObjectsHandler, []string{"DeleteMultipleObjects", "PutBucketPolicy"})
 }
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -680,6 +681,29 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		objectNames = append(objectNames, objectName)
 	}
 
+	for _, name := range []string{"private/object", "public/object"} {
+		// Uploading the object with retention enabled
+		_, err = obj.PutObject(GlobalContext, bucketName, name, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
+		// if object upload fails stop the test.
+		if err != nil {
+			t.Fatalf("Put Object %s:  Error uploading object: <ERROR> %v", name, err)
+		}
+	}
+
+	// The following block will create a bucket policy with delete object to 'public/*'. This is
+	// to test a mixed response of a successful & failure while deleting objects in a single request
+	policyBytes := []byte(fmt.Sprintf(`{"Id": "Policy1637752602639", "Version": "2012-10-17", "Statement": [{"Sid": "Stmt1637752600730", "Action": "s3:DeleteObject", "Effect": "Allow", "Resource": "arn:aws:s3:::%s/public/*", "Principal": "*"}]}`, bucketName))
+	rec := httptest.NewRecorder()
+	req, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)), bytes.NewReader(policyBytes),
+		credentials.AccessKey, credentials.SecretKey, nil)
+	if err != nil {
+		t.Fatalf("Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", err)
+	}
+	apiRouter.ServeHTTP(rec, req)
+	if rec.Code != http.StatusNoContent {
+		t.Errorf("Expected the response status to be `%d`, but instead found `%d`", 200, rec.Code)
+	}
+
 	getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) {
 		for _, objectName := range objectNames {
 			objectList = append(objectList, ObjectToDelete{
@@ -705,6 +729,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	requestList := []DeleteObjectsRequest{
 		{Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])},
 		{Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])},
+		{Quiet: false, Objects: []ObjectToDelete{{ObjectName: "private/object"}, {ObjectName: "public/object"}}},
 	}
 
 	// generate multi objects delete response.
@@ -741,6 +766,20 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	anonResponse := generateMultiDeleteResponse(requestList[0].Quiet, nil, getDeleteErrorList(requestList[0].Objects))
 	encodedAnonResponse := encodeResponse(anonResponse)
 
+	anonRequestWithPartialPublicAccess := encodeResponse(requestList[2])
+	anonResponseWithPartialPublicAccess := generateMultiDeleteResponse(requestList[2].Quiet,
+		[]DeletedObject{
+			{ObjectName: "public/object"},
+		},
+		[]DeleteError{
+			{
+				Code:    errorCodes[ErrAccessDenied].Code,
+				Message: errorCodes[ErrAccessDenied].Description,
+				Key:     "private/object",
+			},
+		})
+	encodedAnonResponseWithPartialPublicAccess := encodeResponse(anonResponseWithPartialPublicAccess)
+
 	testCases := []struct {
 		bucket             string
 		objects            []byte
@@ -800,6 +839,17 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    encodedAnonResponse,
 			expectedRespStatus: http.StatusOK,
 		},
+		// Test case - 6.
+		// Anonymous user has access to some public folder, issue removing with
+		// another private object as well
+		{
+			bucket:             bucketName,
+			objects:            anonRequestWithPartialPublicAccess,
+			accessKey:          "",
+			secretKey:          "",
+			expectedContent:    encodedAnonResponseWithPartialPublicAccess,
+			expectedRespStatus: http.StatusOK,
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/bucket-lifecycle.go

@@ -77,46 +77,81 @@ func NewLifecycleSys() *LifecycleSys {
 type expiryTask struct {
 	objInfo        ObjectInfo
 	versionExpiry  bool
+	restoredObject bool
 }
 
 type expiryState struct {
 	once                sync.Once
-	expiryCh chan expiryTask
+	byDaysCh            chan expiryTask
+	byNewerNoncurrentCh chan newerNoncurrentTask
 }
 
 // PendingTasks returns the number of pending ILM expiry tasks.
 func (es *expiryState) PendingTasks() int {
-	return len(es.expiryCh)
+	return len(es.byDaysCh) + len(es.byNewerNoncurrentCh)
 }
 
-func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
+// close closes work channels exactly once.
+func (es *expiryState) close() {
+	es.once.Do(func() {
+		close(es.byDaysCh)
+		close(es.byNewerNoncurrentCh)
+	})
+}
+
+// enqueueByDays enqueues object versions expired by days for expiry.
+func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersion bool) {
 	select {
 	case <-GlobalContext.Done():
-		es.once.Do(func() {
+		es.close()
-			close(es.expiryCh)
+	case es.byDaysCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}:
-		})
-	case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion}:
 	default:
 	}
 }
 
-var (
+// enqueueByNewerNoncurrent enqueues object versions expired by
-	globalExpiryState *expiryState
+// NewerNoncurrentVersions limit for expiry.
-)
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
+	select {
+	case <-GlobalContext.Done():
+		es.close()
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
+	default:
+	}
+}
+
+var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		expiryCh: make(chan expiryTask, 10000),
+		byDaysCh:            make(chan expiryTask, 10000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
 	}
 }
 
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
 	go func() {
-		for t := range globalExpiryState.expiryCh {
+		for t := range globalExpiryState.byDaysCh {
-			applyExpiryRule(ctx, objectAPI, t.objInfo, false, t.versionExpiry)
+			if t.objInfo.TransitionedObject.Status != "" {
+				applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+			} else {
+				applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+			}
 		}
 	}()
+	go func() {
+		for t := range globalExpiryState.byNewerNoncurrentCh {
+			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+		}
+	}()
+}
+
+// newerNoncurrentTask encapsulates arguments required by worker to expire objects
+// by NewerNoncurrentVersions
+type newerNoncurrentTask struct {
+	bucket   string
+	versions []ObjectToDelete
 }
 
 type transitionState struct {
@@ -215,16 +250,31 @@ var errInvalidStorageClass = errors.New("invalid storage class")
 
 func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 	for _, rule := range lc.Rules {
-		if rule.Transition.StorageClass == "" {
+		if rule.Transition.StorageClass != "" {
-			continue
-		}
 			if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
 				return errInvalidStorageClass
 			}
 		}
+		if rule.NoncurrentVersionTransition.StorageClass != "" {
+			if valid := globalTierConfigMgr.IsTierValid(rule.NoncurrentVersionTransition.StorageClass); !valid {
+				return errInvalidStorageClass
+			}
+		}
+	}
 	return nil
 }
 
+// enqueueTransitionImmediate enqueues obj for transition if eligible.
+// This is to be called after a successful upload of an object (version).
+func enqueueTransitionImmediate(obj ObjectInfo) {
+	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
+		switch lc.ComputeAction(obj.ToLifecycleOpts()) {
+		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+			globalTransitionState.queueTransitionTask(obj)
+		}
+	}
+}
+
 // expireAction represents different actions to be performed on expiry of a
 // restored/transitioned object
 type expireAction int
@@ -247,6 +297,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
 	opts.VersionID = lcOpts.VersionID
+	opts.Expiration = ExpirationOptions{Expire: true}
 	switch action {
 	case expireObj:
 		// When an object is past expiry or when a transitioned object is being
@@ -374,7 +425,7 @@ const (
 
 // Encryption specifies encryption setting on restored bucket
 type Encryption struct {
-	EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"`
+	EncryptionType sse.Algorithm `xml:"EncryptionType"`
 	KMSContext     string        `xml:"KMSContext,omitempty"`
 	KMSKeyID       string        `xml:"KMSKeyId,omitempty"`
 }
@@ -529,7 +580,7 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 
 	if rreq.Type == SelectRestoreRequest {
 		for _, v := range rreq.OutputLocation.S3.UserMetadata {
-			if !strings.HasPrefix("x-amz-meta", strings.ToLower(v.Name)) {
+			if !strings.HasPrefix(strings.ToLower(v.Name), "x-amz-meta") {
 				meta["x-amz-meta-"+v.Name] = v.Value
 				continue
 			}
@@ -707,6 +758,5 @@ func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
 		RestoreOngoing:   oi.RestoreOngoing,
 		RestoreExpires:   oi.RestoreExpires,
 		TransitionStatus: oi.TransitionedObject.Status,
-		RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
 	}
 }

cmd/bucket-listobjects-handlers.go

@@ -32,8 +32,6 @@ import (
 
 func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
 	g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
-	_, cancel := g.WithCancelOnError(ctx)
-	defer cancel()
 	for index := range objects {
 		index := index
 		g.Go(func() error {
@@ -45,7 +43,7 @@ func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
 			return nil
 		}, index)
 	}
-	g.WaitErr()
+	g.Wait()
 }
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode
@@ -61,8 +59,8 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
 	}
 
 	if encodingType != "" {
-		// Only url encoding type is supported
+		// AWS S3 spec only supports 'url' encoding type
-		if strings.ToLower(encodingType) != "url" {
+		if !strings.EqualFold(encodingType, "url") {
 			return ErrInvalidEncodingMethod
 		}
 	}
@@ -128,7 +126,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 10000)
+// This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
@@ -195,7 +193,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 10000)
+// This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
@@ -305,7 +303,7 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 10000)
+// This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //

cmd/bucket-metadata-sys.go

@@ -80,46 +80,8 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		return errServerNotInitialized
 	}
 
-	if globalIsGateway {
+	if globalIsGateway && globalGatewayName != NASBackendGateway {
-		// This code is needed only for gateway implementations.
+		if configFile == bucketPolicyConfig {
-		switch configFile {
-		case bucketSSEConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.EncryptionConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketLifecycleConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.LifecycleConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketTaggingConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.TaggingConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketNotificationConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.NotificationConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketPolicyConfig:
 			if configData == nil {
 				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
 			}
@@ -138,8 +100,13 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 
 	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 	if err != nil {
+		if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
+			// Only single drive mode needs this fallback.
+			meta = newBucketMetadata(bucket)
+		} else {
 			return err
 		}
+	}
 
 	switch configFile {
 	case bucketPolicyConfig:
@@ -447,9 +414,9 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return errServerNotInitialized
 	}
 
-	// In gateway mode, we don't need to load the policies
+	// In gateway mode, we don't need to load bucket metadata except
-	// from the backend.
+	// NAS gateway backend.
-	if globalIsGateway {
+	if globalIsGateway && !objAPI.IsNotificationSupported() {
 		return nil
 	}
 
@@ -470,11 +437,20 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			})
 			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
 			if err != nil {
+				if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
+					meta = newBucketMetadata(buckets[index].Name)
+				} else {
 					return err
 				}
+			}
 			sys.Lock()
 			sys.metadataMap[buckets[index].Name] = meta
 			sys.Unlock()
+
+			globalNotificationSys.set(buckets[index], meta) // set notification targets
+
+			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
+
 			return nil
 		}, index)
 	}

cmd/bucket-notification-handlers.go

@@ -72,8 +72,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	config.SetRegion(globalServerRegion)
+	config.SetRegion(globalSite.Region)
-	if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
+	if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
 		if ok {
 			for i, queue := range config.QueueList {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerRegion, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {

cmd/bucket-object-lock.go

@@ -93,7 +93,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 	if gerr != nil { // error from GetObjectInfo
 		switch gerr.(type) {
 		case MethodNotAllowed: // This happens usually for a delete marker
-			if oi.DeleteMarker {
+			if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 				// Delete marker should be present and valid.
 				return ErrNone
 			}
@@ -175,68 +175,76 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
-func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) (ObjectInfo, APIErrorCode) {
+func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi ObjectInfo, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) error {
 	byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header)
-	opts, err := getOpts(ctx, r, bucket, object)
-	if err != nil {
-		return ObjectInfo{}, toAPIErrorCode(ctx, err)
-	}
-
-	oi, err := getObjectInfoFn(ctx, bucket, object, opts)
-	if err != nil {
-		return oi, toAPIErrorCode(ctx, err)
-	}
 
 	t, err := objectlock.UTCNowNTP()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return oi, ErrObjectLocked
+		return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 	}
 
-	// Pass in relative days from current time, to additionally to verify "object-lock-remaining-retention-days" policy if any.
+	// Pass in relative days from current time, to additionally
+	// to verify "object-lock-remaining-retention-days" policy if any.
 	days := int(math.Ceil(math.Abs(objRetention.RetainUntilDate.Sub(t).Hours()) / 24))
 
 	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
 	if ret.Mode.Valid() {
 		// Retention has expired you may change whatever you like.
 		if ret.RetainUntilDate.Before(t) {
-			perm := isPutRetentionAllowed(bucket, object,
+			apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 				days, objRetention.RetainUntilDate.Time,
 				objRetention.Mode, byPassSet, r, cred,
 				owner)
-			return oi, perm
+			switch apiErr {
+			case ErrAccessDenied:
+				return errAuthentication
+			}
+			return nil
 		}
 
 		switch ret.Mode {
 		case objectlock.RetGovernance:
-			govPerm := isPutRetentionAllowed(bucket, object, days,
+			govPerm := isPutRetentionAllowed(oi.Bucket, oi.Name, days,
 				objRetention.RetainUntilDate.Time, objRetention.Mode,
 				byPassSet, r, cred, owner)
 			// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
 			if !byPassSet {
 				if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
-					return oi, ErrObjectLocked
+					return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 				}
 			}
-			return oi, govPerm
+			switch govPerm {
+			case ErrAccessDenied:
+				return errAuthentication
+			}
+			return nil
 		case objectlock.RetCompliance:
 			// Compliance retention mode cannot be changed or shortened.
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
 			if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
-				return oi, ErrObjectLocked
+				return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 			}
-			compliancePerm := isPutRetentionAllowed(bucket, object,
+			apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 				days, objRetention.RetainUntilDate.Time, objRetention.Mode,
 				false, r, cred, owner)
-			return oi, compliancePerm
+			switch apiErr {
+			case ErrAccessDenied:
+				return errAuthentication
 			}
-		return oi, ErrNone
+			return nil
+		}
+		return nil
 	} // No pre-existing retention metadata present.
 
-	perm := isPutRetentionAllowed(bucket, object,
+	apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 		days, objRetention.RetainUntilDate.Time,
 		objRetention.Mode, byPassSet, r, cred, owner)
-	return oi, perm
+	switch apiErr {
+	case ErrAccessDenied:
+		return errAuthentication
+	}
+	return nil
 }
 
 // checkPutObjectLockAllowed enforces object retention policy and legal hold policy

cmd/bucket-policy-handlers.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -76,7 +79,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
+	bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -99,6 +108,16 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
+	// Call site replication hook.
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypePolicy,
+		Bucket: bucket,
+		Policy: bucketPolicyBytes,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Success.
 	writeSuccessNoContent(w)
 }
@@ -134,6 +153,15 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypePolicy,
+		Bucket: bucket,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Success.
 	writeSuccessNoContent(w)
 }

cmd/bucket-policy.go

@@ -172,11 +172,12 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		vStr, ok := v.(string)
 		if ok {
 			// Special case for AD/LDAP STS users
-			if k == ldapUser {
+			switch k {
+			case ldapUser:
 				args["user"] = []string{vStr}
-			} else if k == ldapUserN {
+			case ldapUserN:
 				args["username"] = []string{vStr}
-			} else {
+			default:
 				args[k] = []string{vStr}
 			}
 		}

cmd/bucket-quota.go

@@ -89,7 +89,10 @@ func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64)
 			return err
 		}
 
-		dui := v.(madmin.DataUsageInfo)
+		dui, ok := v.(DataUsageInfo)
+		if !ok {
+			return fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
+		}
 
 		bui, ok := dui.BucketsUsage[bucket]
 		if !ok {
@@ -116,7 +119,7 @@ func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
 
 // enforceFIFOQuota deletes objects in FIFO order until sufficient objects
 // have been deleted so as to bring bucket usage within quota.
-func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui madmin.BucketUsageInfo) {
+func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui BucketUsageInfo) {
 	// Check if the current bucket has quota restrictions, if not skip it
 	cfg, err := globalBucketQuotaSys.Get(bucket)
 	if err != nil {

cmd/bucket-replication-stats.go

@@ -20,16 +20,18 @@ package cmd
 import (
 	"context"
 	"sync"
-	"sync/atomic"
+	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
 )
 
 func (b *BucketReplicationStats) hasReplicationUsage() bool {
-	return b.FailedSize > 0 ||
+	for _, s := range b.Stats {
-		b.ReplicatedSize > 0 ||
+		if s.hasReplicationUsage() {
-		b.ReplicaSize > 0 ||
+			return true
-		b.FailedCount > 0
+		}
+	}
+	return false
 }
 
 // ReplicationStats holds the global in-memory replication stats
@@ -37,6 +39,7 @@ type ReplicationStats struct {
 	Cache      map[string]*BucketReplicationStats
 	UsageCache map[string]*BucketReplicationStats
 	sync.RWMutex
+	ulock sync.RWMutex
 }
 
 // Delete deletes in-memory replication statistics for a bucket.
@@ -48,50 +51,75 @@ func (r *ReplicationStats) Delete(bucket string) {
 	r.Lock()
 	defer r.Unlock()
 	delete(r.Cache, bucket)
+
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
 	delete(r.UsageCache, bucket)
 
 }
 
-// Update updates in-memory replication statistics with new values.
+// UpdateReplicaStat updates in-memory replica statistics with new values.
-func (r *ReplicationStats) Update(bucket string, n int64, status, prevStatus replication.StatusType, opType replication.Type) {
+func (r *ReplicationStats) UpdateReplicaStat(bucket string, n int64) {
 	if r == nil {
 		return
 	}
 
-	r.RLock()
+	r.Lock()
-	b, ok := r.Cache[bucket]
+	defer r.Unlock()
+	bs, ok := r.Cache[bucket]
 	if !ok {
-		b = &BucketReplicationStats{}
+		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+	}
+	bs.ReplicaSize += n
+	r.Cache[bucket] = bs
+}
+
+// Update updates in-memory replication statistics with new values.
+func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration time.Duration, status, prevStatus replication.StatusType, opType replication.Type) {
+	if r == nil {
+		return
+	}
+	r.Lock()
+	defer r.Unlock()
+
+	bs, ok := r.Cache[bucket]
+	if !ok {
+		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		r.Cache[bucket] = bs
+	}
+	b, ok := bs.Stats[arn]
+	if !ok {
+		b = &BucketReplicationStat{}
+		bs.Stats[arn] = b
 	}
-	r.RUnlock()
 	switch status {
 	case replication.Completed:
 		switch prevStatus { // adjust counters based on previous state
 		case replication.Failed:
-			atomic.AddUint64(&b.FailedCount, ^uint64(0))
+			b.FailedCount--
 		}
 		if opType == replication.ObjectReplicationType {
-			atomic.AddUint64(&b.ReplicatedSize, uint64(n))
+			b.ReplicatedSize += n
 			switch prevStatus {
 			case replication.Failed:
-				atomic.AddUint64(&b.FailedSize, ^uint64(n-1))
+				b.FailedSize -= n
+			}
+			if duration > 0 {
+				b.Latency.update(n, duration)
 			}
 		}
 	case replication.Failed:
 		if opType == replication.ObjectReplicationType {
 			if prevStatus == replication.Pending {
-				atomic.AddUint64(&b.FailedSize, uint64(n))
+				b.FailedSize += n
-				atomic.AddUint64(&b.FailedCount, 1)
+				b.FailedCount++
 			}
 		}
 	case replication.Replica:
 		if opType == replication.ObjectReplicationType {
-			atomic.AddUint64(&b.ReplicaSize, uint64(n))
+			b.ReplicaSize += n
 		}
 	}
-	r.Lock()
-	r.Cache[bucket] = b
-	r.Unlock()
 }
 
 // GetInitialUsage get replication metrics available at the time of cluster initialization
@@ -99,26 +127,19 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	if r == nil {
 		return BucketReplicationStats{}
 	}
-
+	r.ulock.RLock()
-	r.RLock()
+	defer r.ulock.RUnlock()
-	defer r.RUnlock()
-
 	st, ok := r.UsageCache[bucket]
 	if !ok {
 		return BucketReplicationStats{}
 	}
-	return BucketReplicationStats{
+	return st.Clone()
-		FailedSize:     atomic.LoadUint64(&st.FailedSize),
-		ReplicatedSize: atomic.LoadUint64(&st.ReplicatedSize),
-		ReplicaSize:    atomic.LoadUint64(&st.ReplicaSize),
-		FailedCount:    atomic.LoadUint64(&st.FailedCount),
-	}
 }
 
 // Get replication metrics for a bucket from this node since this node came up.
 func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	if r == nil {
-		return BucketReplicationStats{}
+		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
 	}
 
 	r.RLock()
@@ -128,43 +149,61 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	if !ok {
 		return BucketReplicationStats{}
 	}
-
+	return st.Clone()
-	return BucketReplicationStats{
-		FailedSize:     atomic.LoadUint64(&st.FailedSize),
-		ReplicatedSize: atomic.LoadUint64(&st.ReplicatedSize),
-		ReplicaSize:    atomic.LoadUint64(&st.ReplicaSize),
-		FailedCount:    atomic.LoadUint64(&st.FailedCount),
-	}
 }
 
 // NewReplicationStats initialize in-memory replication statistics
 func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *ReplicationStats {
-	st := &ReplicationStats{
+	return &ReplicationStats{
 		Cache:      make(map[string]*BucketReplicationStats),
 		UsageCache: make(map[string]*BucketReplicationStats),
 	}
+}
 
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+// load replication metrics at cluster start from initial data usage
+func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
+	rTimer := time.NewTimer(time.Minute * 1)
+	defer rTimer.Stop()
+	var (
+		dui DataUsageInfo
+		err error
+	)
+outer:
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-rTimer.C:
+			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
 			if err != nil {
-		return st
+				continue
+			}
+			// If LastUpdate is set, data usage is available.
+			if !dui.LastUpdate.IsZero() {
+				break outer
+			}
+		}
 	}
 
-	// data usage has not captured any data yet.
+	m := make(map[string]*BucketReplicationStats)
-	if dataUsageInfo.LastUpdate.IsZero() {
+	for bucket, usage := range dui.BucketsUsage {
-		return st
-	}
-
-	for bucket, usage := range dataUsageInfo.BucketsUsage {
 		b := &BucketReplicationStats{
-			FailedSize:     usage.ReplicationFailedSize,
+			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
-			ReplicatedSize: usage.ReplicatedSize,
-			ReplicaSize:    usage.ReplicaSize,
-			FailedCount:    usage.ReplicationFailedCount,
 		}
+		for arn, uinfo := range usage.ReplicationInfo {
+			b.Stats[arn] = &BucketReplicationStat{
+				FailedSize:     int64(uinfo.ReplicationFailedSize),
+				ReplicatedSize: int64(uinfo.ReplicatedSize),
+				ReplicaSize:    int64(uinfo.ReplicaSize),
+				FailedCount:    int64(uinfo.ReplicationFailedCount),
+			}
+		}
+		b.ReplicaSize += int64(usage.ReplicaSize)
 		if b.hasReplicationUsage() {
-			st.UsageCache[bucket] = b
+			m[bucket] = b
 		}
 	}
-
+	r.ulock.Lock()
-	return st
+	defer r.ulock.Unlock()
+	r.UsageCache = m
 }

cmd/bucket-replication-utils.go

@@ -0,0 +1,603 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/replication"
+)
+
+//go:generate msgp -file=$GOFILE
+
+// replicatedTargetInfo struct represents replication info on a target
+type replicatedTargetInfo struct {
+	Arn                   string
+	Size                  int64
+	Duration              time.Duration
+	ReplicationAction     replicationAction // full or metadata only
+	OpType                replication.Type  // whether incoming replication, existing object, healing etc..
+	ReplicationStatus     replication.StatusType
+	PrevReplicationStatus replication.StatusType
+	VersionPurgeStatus    VersionPurgeStatusType
+	ResyncTimestamp       string
+	ReplicationResynced   bool // true only if resync attempted for this target
+}
+
+// Empty returns true for a target if arn is empty
+func (rt replicatedTargetInfo) Empty() bool {
+	return rt.Arn == ""
+}
+
+type replicatedInfos struct {
+	ReplicationTimeStamp time.Time
+	Targets              []replicatedTargetInfo
+}
+
+func (ri replicatedInfos) CompletedSize() (sz int64) {
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		if t.ReplicationStatus == replication.Completed && t.PrevReplicationStatus != replication.Completed {
+			sz += t.Size
+		}
+	}
+	return sz
+}
+
+// ReplicationAttempted returns true if replication was attempted on any of the targets for the object version
+// queued
+func (ri replicatedInfos) ReplicationResynced() bool {
+	for _, t := range ri.Targets {
+		if t.Empty() || !t.ReplicationResynced {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+func (ri replicatedInfos) ReplicationStatusInternal() string {
+	b := new(bytes.Buffer)
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		fmt.Fprintf(b, "%s=%s;", t.Arn, t.ReplicationStatus.String())
+	}
+	return b.String()
+}
+
+func (ri replicatedInfos) ReplicationStatus() replication.StatusType {
+	if len(ri.Targets) == 0 {
+		return replication.StatusType("")
+	}
+	completed := 0
+	for _, v := range ri.Targets {
+		switch v.ReplicationStatus {
+		case replication.Failed:
+			return replication.Failed
+		case replication.Completed:
+			completed++
+		}
+	}
+	if completed == len(ri.Targets) {
+		return replication.Completed
+	}
+	return replication.Pending
+}
+
+func (ri replicatedInfos) VersionPurgeStatus() VersionPurgeStatusType {
+	if len(ri.Targets) == 0 {
+		return VersionPurgeStatusType("")
+	}
+	completed := 0
+	for _, v := range ri.Targets {
+		switch v.VersionPurgeStatus {
+		case Failed:
+			return Failed
+		case Complete:
+			completed++
+		}
+	}
+	if completed == len(ri.Targets) {
+		return Complete
+	}
+	return Pending
+}
+
+func (ri replicatedInfos) VersionPurgeStatusInternal() string {
+	b := new(bytes.Buffer)
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		if t.VersionPurgeStatus.Empty() {
+			continue
+		}
+		fmt.Fprintf(b, "%s=%s;", t.Arn, t.VersionPurgeStatus)
+	}
+	return b.String()
+}
+
+func (ri replicatedInfos) Action() replicationAction {
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		// rely on replication action from target that actually performed replication now.
+		if t.PrevReplicationStatus != replication.Completed {
+			return t.ReplicationAction
+		}
+	}
+	return replicateNone
+}
+
+var replStatusRegex = regexp.MustCompile(`([^=].*?)=([^,].*?);`)
+
+// TargetReplicationStatus - returns replication status of a target
+func (o *ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(o.ReplicationStatusInternal, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			return
+		}
+		if repStatMatch[1] == arn {
+			return replication.StatusType(repStatMatch[2])
+		}
+	}
+	return
+}
+
+type replicateTargetDecision struct {
+	Replicate   bool   // Replicate to this target
+	Synchronous bool   // Synchronous replication configured.
+	Arn         string // ARN of replication target
+	ID          string
+}
+
+func (t *replicateTargetDecision) String() string {
+	return fmt.Sprintf("%t;%t;%s;%s", t.Replicate, t.Synchronous, t.Arn, t.ID)
+}
+
+func newReplicateTargetDecision(arn string, replicate bool, sync bool) replicateTargetDecision {
+	d := replicateTargetDecision{
+		Replicate:   replicate,
+		Synchronous: sync,
+		Arn:         arn,
+	}
+	return d
+}
+
+// ReplicateDecision represents replication decision for each target
+type ReplicateDecision struct {
+	targetsMap map[string]replicateTargetDecision
+}
+
+// ReplicateAny returns true if atleast one target qualifies for replication
+func (d *ReplicateDecision) ReplicateAny() bool {
+	for _, t := range d.targetsMap {
+		if t.Replicate {
+			return true
+		}
+	}
+	return false
+}
+
+// Synchronous returns true if atleast one target qualifies for synchronous replication
+func (d *ReplicateDecision) Synchronous() bool {
+	for _, t := range d.targetsMap {
+		if t.Synchronous {
+			return true
+		}
+	}
+	return false
+}
+
+func (d *ReplicateDecision) String() string {
+	b := new(bytes.Buffer)
+	for key, value := range d.targetsMap {
+		fmt.Fprintf(b, "%s=%s,", key, value.String())
+	}
+	return strings.TrimSuffix(b.String(), ",")
+}
+
+// Set updates ReplicateDecision with target's replication decision
+func (d *ReplicateDecision) Set(t replicateTargetDecision) {
+	if d.targetsMap == nil {
+		d.targetsMap = make(map[string]replicateTargetDecision)
+	}
+	d.targetsMap[t.Arn] = t
+}
+
+// PendingStatus returns a stringified representation of internal replication status with all targets marked as `PENDING`
+func (d *ReplicateDecision) PendingStatus() string {
+	b := new(bytes.Buffer)
+	for _, k := range d.targetsMap {
+		if k.Replicate {
+			fmt.Fprintf(b, "%s=%s;", k.Arn, replication.Pending.String())
+		}
+	}
+	return b.String()
+}
+
+// ResyncDecision is a struct representing a map with target's individual resync decisions
+type ResyncDecision struct {
+	targets map[string]ResyncTargetDecision
+}
+
+// Empty returns true if no targets with resync decision present
+func (r *ResyncDecision) Empty() bool {
+	return r.targets == nil
+}
+
+func (r *ResyncDecision) mustResync() bool {
+	for _, v := range r.targets {
+		if v.Replicate {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *ResyncDecision) mustResyncTarget(tgtArn string) bool {
+	if r.targets == nil {
+		return false
+	}
+	v, ok := r.targets[tgtArn]
+	if ok && v.Replicate {
+		return true
+	}
+	return false
+}
+
+// ResyncTargetDecision is struct that represents resync decision for this target
+type ResyncTargetDecision struct {
+	Replicate       bool
+	ResetID         string
+	ResetBeforeDate time.Time
+}
+
+var errInvalidReplicateDecisionFormat = fmt.Errorf("ReplicateDecision has invalid format")
+
+// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
+// ReplicateDecision struct
+func parseReplicateDecision(s string) (r ReplicateDecision, err error) {
+	r = ReplicateDecision{
+		targetsMap: make(map[string]replicateTargetDecision),
+	}
+	if len(s) == 0 {
+		return
+	}
+	pairs := strings.Split(s, ",")
+	for _, p := range pairs {
+		slc := strings.Split(p, "=")
+		if len(slc) != 2 {
+			return r, errInvalidReplicateDecisionFormat
+		}
+		tgtStr := strings.TrimPrefix(slc[1], "\"")
+		tgtStr = strings.TrimSuffix(tgtStr, "\"")
+		tgt := strings.Split(tgtStr, ";")
+		if len(tgt) != 4 {
+			return r, errInvalidReplicateDecisionFormat
+		}
+		var replicate, sync bool
+		var err error
+		replicate, err = strconv.ParseBool(tgt[0])
+		if err != nil {
+			return r, err
+		}
+		sync, err = strconv.ParseBool(tgt[1])
+		if err != nil {
+			return r, err
+		}
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3]}
+	}
+	return
+}
+
+// ReplicationState represents internal replication state
+type ReplicationState struct {
+	ReplicaTimeStamp          time.Time              // timestamp when last replica update was received
+	ReplicaStatus             replication.StatusType // replica statusstringis
+	DeleteMarker              bool                   // represents DeleteMarker replication state
+	ReplicationTimeStamp      time.Time              // timestamp when last replication activity happened
+	ReplicationStatusInternal string                 // stringified representation of all replication activity
+	// VersionPurgeStatusInternal is internally in the format "arn1=PENDING;arn2=COMMPLETED;"
+	VersionPurgeStatusInternal string                            // stringified representation of all version purge statuses
+	ReplicateDecisionStr       string                            // stringified representation of replication decision for each target
+	Targets                    map[string]replication.StatusType // map of ARN->replication status for ongoing replication activity
+	PurgeTargets               map[string]VersionPurgeStatusType // map of ARN->VersionPurgeStatus for all the targets
+	ResetStatusesMap           map[string]string                 // map of ARN-> stringified reset id and timestamp for all the targets
+}
+
+// Equal returns true if replication state is identical for version purge statuses and (replica)tion statuses.
+func (rs *ReplicationState) Equal(o ReplicationState) bool {
+	return rs.ReplicaStatus == o.ReplicaStatus &&
+		rs.ReplicaTimeStamp.Equal(o.ReplicaTimeStamp) &&
+		rs.ReplicationTimeStamp.Equal(o.ReplicationTimeStamp) &&
+		rs.ReplicationStatusInternal == o.ReplicationStatusInternal &&
+		rs.VersionPurgeStatusInternal == o.VersionPurgeStatusInternal
+}
+
+// CompositeReplicationStatus returns overall replication status for the object version being replicated.
+func (rs *ReplicationState) CompositeReplicationStatus() (st replication.StatusType) {
+	switch {
+	case rs.ReplicationStatusInternal != "":
+		switch replication.StatusType(rs.ReplicationStatusInternal) {
+		case replication.Pending, replication.Completed, replication.Failed, replication.Replica: // for backward compatibility
+			return replication.StatusType(rs.ReplicationStatusInternal)
+		default:
+			replStatus := getCompositeReplicationStatus(rs.Targets)
+			// return REPLICA status if replica received timestamp is later than replication timestamp
+			// provided object replication completed for all targets.
+			if !rs.ReplicaTimeStamp.Equal(timeSentinel) && replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) {
+				return rs.ReplicaStatus
+			}
+			return replStatus
+		}
+	case !rs.ReplicaStatus.Empty():
+		return rs.ReplicaStatus
+	default:
+		return
+	}
+}
+
+// CompositeVersionPurgeStatus returns overall replication purge status for the permanent delete being replicated.
+func (rs *ReplicationState) CompositeVersionPurgeStatus() VersionPurgeStatusType {
+	switch VersionPurgeStatusType(rs.VersionPurgeStatusInternal) {
+	case Pending, Complete, Failed: // for backward compatibility
+		return VersionPurgeStatusType(rs.VersionPurgeStatusInternal)
+	default:
+		return getCompositeVersionPurgeStatus(rs.PurgeTargets)
+	}
+}
+
+// TargetState returns replicatedInfos struct initialized with the previous state of replication
+func (rs *ReplicationState) targetState(arn string) (r replicatedTargetInfo) {
+	return replicatedTargetInfo{
+		Arn:                   arn,
+		PrevReplicationStatus: rs.Targets[arn],
+		VersionPurgeStatus:    rs.PurgeTargets[arn],
+		ResyncTimestamp:       rs.ResetStatusesMap[arn],
+	}
+}
+
+// getReplicationState returns replication state using target replicated info for the targets
+func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID string) ReplicationState {
+	rs := ReplicationState{
+		ReplicateDecisionStr: prevState.ReplicateDecisionStr,
+		ResetStatusesMap:     prevState.ResetStatusesMap,
+		ReplicaTimeStamp:     prevState.ReplicaTimeStamp,
+		ReplicaStatus:        prevState.ReplicaStatus,
+	}
+	var replStatuses, vpurgeStatuses string
+	replStatuses = rinfos.ReplicationStatusInternal()
+	rs.Targets = replicationStatusesMap(replStatuses)
+	rs.ReplicationStatusInternal = replStatuses
+	rs.ReplicationTimeStamp = rinfos.ReplicationTimeStamp
+
+	vpurgeStatuses = rinfos.VersionPurgeStatusInternal()
+	rs.VersionPurgeStatusInternal = vpurgeStatuses
+	rs.PurgeTargets = versionPurgeStatusesMap(vpurgeStatuses)
+
+	for _, rinfo := range rinfos.Targets {
+		if rinfo.ResyncTimestamp != "" {
+			rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
+		}
+	}
+	return rs
+}
+
+// constructs a replication status map from string representation
+func replicationStatusesMap(s string) map[string]replication.StatusType {
+	targets := make(map[string]replication.StatusType)
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(s, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			continue
+		}
+		status := replication.StatusType(repStatMatch[2])
+		targets[repStatMatch[1]] = status
+	}
+	return targets
+}
+
+// constructs a version purge status map from string representation
+func versionPurgeStatusesMap(s string) map[string]VersionPurgeStatusType {
+	targets := make(map[string]VersionPurgeStatusType)
+	purgeStatusMatches := replStatusRegex.FindAllStringSubmatch(s, -1)
+	for _, purgeStatusMatch := range purgeStatusMatches {
+		if len(purgeStatusMatch) != 3 {
+			continue
+		}
+		targets[purgeStatusMatch[1]] = VersionPurgeStatusType(purgeStatusMatch[2])
+	}
+	return targets
+}
+
+// return the overall replication status for all the targets
+func getCompositeReplicationStatus(m map[string]replication.StatusType) replication.StatusType {
+	if len(m) == 0 {
+		return replication.StatusType("")
+	}
+	completed := 0
+	for _, v := range m {
+		switch v {
+		case replication.Failed:
+			return replication.Failed
+		case replication.Completed:
+			completed++
+		}
+	}
+	if completed == len(m) {
+		return replication.Completed
+	}
+	return replication.Pending
+}
+
+// return the overall version purge status for all the targets
+func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) VersionPurgeStatusType {
+	if len(m) == 0 {
+		return VersionPurgeStatusType("")
+	}
+	completed := 0
+	for _, v := range m {
+		switch v {
+		case Failed:
+			return Failed
+		case Complete:
+			completed++
+		}
+	}
+	if completed == len(m) {
+		return Complete
+	}
+	return Pending
+}
+
+// getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo
+func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
+	oi := objInfo.Clone()
+	if rcfg.Config != nil && rcfg.Config.RoleArn != "" {
+		// For backward compatibility of objects pending/failed replication.
+		// Save replication related statuses in the new internal representation for
+		// compatible behavior.
+		if !oi.ReplicationStatus.Empty() {
+			oi.ReplicationStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.ReplicationStatus)
+		}
+		if !oi.VersionPurgeStatus.Empty() {
+			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
+		}
+		for k, v := range oi.UserDefined {
+			switch {
+			case strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset):
+				delete(oi.UserDefined, k)
+				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
+			}
+		}
+	}
+	var dsc ReplicateDecision
+	var tgtStatuses map[string]replication.StatusType
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
+			ObjectName: oi.Name,
+			VersionID:  oi.VersionID,
+		}, oi, ObjectOptions{}, nil)
+	} else {
+		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
+			UserDefined: oi.UserDefined,
+		}, replication.HealReplicationType, ObjectOptions{}))
+	}
+	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
+
+	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
+
+	return ReplicateObjectInfo{
+		ObjectInfo:        oi,
+		OpType:            replication.HealReplicationType,
+		Dsc:               dsc,
+		ExistingObjResync: existingObjResync,
+		TargetStatuses:    tgtStatuses,
+	}
+}
+
+// vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
+func (o *ObjectInfo) getReplicationState(dsc string, vID string, heal bool) ReplicationState {
+	rs := ReplicationState{
+		ReplicationStatusInternal:  o.ReplicationStatusInternal,
+		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
+		ReplicateDecisionStr:       dsc,
+		Targets:                    make(map[string]replication.StatusType),
+		PurgeTargets:               make(map[string]VersionPurgeStatusType),
+		ResetStatusesMap:           make(map[string]string),
+	}
+	rs.Targets = replicationStatusesMap(o.ReplicationStatusInternal)
+	rs.PurgeTargets = versionPurgeStatusesMap(o.VersionPurgeStatusInternal)
+	for k, v := range o.UserDefined {
+		if strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset) {
+			arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset))
+			rs.ResetStatusesMap[arn] = v
+		}
+	}
+	return rs
+}
+
+// ReplicationState returns replication state using other internal replication metadata in ObjectToDelete
+func (o *ObjectToDelete) ReplicationState() ReplicationState {
+	r := ReplicationState{
+		ReplicationStatusInternal:  o.DeleteMarkerReplicationStatus,
+		VersionPurgeStatusInternal: o.VersionPurgeStatuses,
+		ReplicateDecisionStr:       o.ReplicateDecisionStr,
+	}
+
+	r.Targets = replicationStatusesMap(o.DeleteMarkerReplicationStatus)
+	r.PurgeTargets = versionPurgeStatusesMap(o.VersionPurgeStatuses)
+	return r
+}
+
+// VersionPurgeStatus returns a composite version purge status across targets
+func (d *DeletedObject) VersionPurgeStatus() VersionPurgeStatusType {
+	return d.ReplicationState.CompositeVersionPurgeStatus()
+}
+
+// DeleteMarkerReplicationStatus return composite replication status of delete marker across targets
+func (d *DeletedObject) DeleteMarkerReplicationStatus() replication.StatusType {
+	return d.ReplicationState.CompositeReplicationStatus()
+}
+
+// ResyncTargetsInfo holds a slice of targets with resync info per target
+type ResyncTargetsInfo struct {
+	Targets []ResyncTarget `json:"target,omitempty"`
+}
+
+// ResyncTarget is a struct representing the Target reset ID where target is identified by its Arn
+type ResyncTarget struct {
+	Arn     string `json:"arn"`
+	ResetID string `json:"resetid"`
+}
+
+// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
+type VersionPurgeStatusType string
+
+const (
+	// Pending - versioned delete replication is pending.
+	Pending VersionPurgeStatusType = "PENDING"
+
+	// Complete - versioned delete replication is now complete, erase version on disk.
+	Complete VersionPurgeStatusType = "COMPLETE"
+
+	// Failed - versioned delete replication failed.
+	Failed VersionPurgeStatusType = "FAILED"
+)
+
+// Empty returns true if purge status was not set.
+func (v VersionPurgeStatusType) Empty() bool {
+	return string(v) == ""
+}
+
+// Pending returns true if the version is pending purge.
+func (v VersionPurgeStatusType) Pending() bool {
+	return v == Pending || v == Failed
+}

cmd/bucket-replication-utils_gen_test.go

@@ -0,0 +1,688 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
+	v := ReplicateDecision{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicateDecision(t *testing.T) {
+	v := ReplicateDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicateDecision Msgsize() is inaccurate")
+	}
+
+	vn := ReplicateDecision{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalReplicationState(t *testing.T) {
+	v := ReplicationState{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationState(t *testing.T) {
+	v := ReplicationState{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationState Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationState{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncDecision(t *testing.T) {
+	v := ResyncDecision{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncDecision(t *testing.T) {
+	v := ResyncDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncDecision Msgsize() is inaccurate")
+	}
+
+	vn := ResyncDecision{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncTarget(t *testing.T) {
+	v := ResyncTarget{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncTarget(t *testing.T) {
+	v := ResyncTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncTarget Msgsize() is inaccurate")
+	}
+
+	vn := ResyncTarget{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncTargetDecision(t *testing.T) {
+	v := ResyncTargetDecision{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncTargetDecision(t *testing.T) {
+	v := ResyncTargetDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncTargetDecision Msgsize() is inaccurate")
+	}
+
+	vn := ResyncTargetDecision{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncTargetsInfo(t *testing.T) {
+	v := ResyncTargetsInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncTargetsInfo(t *testing.T) {
+	v := ResyncTargetsInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncTargetsInfo Msgsize() is inaccurate")
+	}
+
+	vn := ResyncTargetsInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-replication-utils_test.go

@@ -0,0 +1,246 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+
+	"github.com/minio/minio/internal/bucket/replication"
+)
+
+var replicatedInfosTests = []struct {
+	name                              string
+	tgtInfos                          []replicatedTargetInfo
+	expectedCompletedSize             int64
+	expectedReplicationStatusInternal string
+	expectedReplicationStatus         replication.StatusType
+	expectedOpType                    replication.Type
+	expectedAction                    replicationAction
+}{
+	{ // 1. empty tgtInfos slice
+		name:                              "no replicated targets",
+		tgtInfos:                          []replicatedTargetInfo{},
+		expectedCompletedSize:             0,
+		expectedReplicationStatusInternal: "",
+		expectedReplicationStatus:         replication.StatusType(""),
+		expectedOpType:                    replication.UnsetReplicationType,
+		expectedAction:                    replicateNone,
+	},
+	{ // 2. replication completed to single target
+		name: "replication completed to single target",
+		tgtInfos: []replicatedTargetInfo{
+			{
+				Arn:                   "arn1",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Completed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			},
+		},
+		expectedCompletedSize:             249,
+		expectedReplicationStatusInternal: "arn1=COMPLETED;",
+		expectedReplicationStatus:         replication.Completed,
+		expectedOpType:                    replication.ObjectReplicationType,
+		expectedAction:                    replicateAll,
+	},
+	{ // 3. replication completed to single target; failed to another
+		name: "replication completed to single target",
+		tgtInfos: []replicatedTargetInfo{
+			{
+				Arn:                   "arn1",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Completed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			},
+			{
+				Arn:                   "arn2",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Failed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			}},
+		expectedCompletedSize:             249,
+		expectedReplicationStatusInternal: "arn1=COMPLETED;arn2=FAILED;",
+		expectedReplicationStatus:         replication.Failed,
+		expectedOpType:                    replication.ObjectReplicationType,
+		expectedAction:                    replicateAll,
+	},
+	{ // 4. replication pending on one target; failed to another
+		name: "replication completed to single target",
+		tgtInfos: []replicatedTargetInfo{
+			{
+				Arn:                   "arn1",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Pending,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			},
+			{
+				Arn:                   "arn2",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Failed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			}},
+		expectedCompletedSize:             0,
+		expectedReplicationStatusInternal: "arn1=PENDING;arn2=FAILED;",
+		expectedReplicationStatus:         replication.Failed,
+		expectedOpType:                    replication.ObjectReplicationType,
+		expectedAction:                    replicateAll,
+	},
+}
+
+func TestReplicatedInfos(t *testing.T) {
+	for i, test := range replicatedInfosTests {
+		rinfos := replicatedInfos{
+			Targets: test.tgtInfos,
+		}
+		if actualSize := rinfos.CompletedSize(); actualSize != test.expectedCompletedSize {
+			t.Errorf("Test%d (%s): Size  got %d , want %d", i+1, test.name, actualSize, test.expectedCompletedSize)
+		}
+		if repStatusStr := rinfos.ReplicationStatusInternal(); repStatusStr != test.expectedReplicationStatusInternal {
+			t.Errorf("Test%d (%s): Internal replication status  got %s , want %s", i+1, test.name, repStatusStr, test.expectedReplicationStatusInternal)
+		}
+		if repStatus := rinfos.ReplicationStatus(); repStatus != test.expectedReplicationStatus {
+			t.Errorf("Test%d (%s): ReplicationStatus  got %s , want %s", i+1, test.name, repStatus, test.expectedReplicationStatus)
+		}
+		if action := rinfos.Action(); action != test.expectedAction {
+			t.Errorf("Test%d (%s): Action  got %s , want %s", i+1, test.name, action, test.expectedAction)
+		}
+	}
+}
+
+var parseReplicationDecisionTest = []struct {
+	name   string
+	dsc    string
+	expDsc ReplicateDecision
+	expErr error
+}{
+	{ // 1.
+		name: "empty string",
+		dsc:  "",
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{},
+		},
+		expErr: nil,
+	},
+
+	{ // 2.
+		name:   "replicate decision for one target",
+		dsc:    "arn:minio:replication::id:bucket=true;false;arn:minio:replication::id:bucket;id",
+		expErr: nil,
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{
+				"arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false),
+			},
+		},
+	},
+	{ // 3.
+		name:   "replicate decision for multiple targets",
+		dsc:    "arn:minio:replication::id:bucket=true;false;arn:minio:replication::id:bucket;id,arn:minio:replication::id2:bucket=false;true;arn:minio:replication::id2:bucket;id2",
+		expErr: nil,
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{
+				"arn:minio:replication::id:bucket":  newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false),
+				"arn:minio:replication::id2:bucket": newReplicateTargetDecision("arn:minio:replication::id2:bucket", false, true),
+			},
+		},
+	},
+	{ // 4.
+		name:   "invalid format replicate decision for one target",
+		dsc:    "arn:minio:replication::id:bucket:true;false;arn:minio:replication::id:bucket;id",
+		expErr: errInvalidReplicateDecisionFormat,
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{
+				"arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false),
+			},
+		},
+	},
+}
+
+func TestParseReplicateDecision(t *testing.T) {
+	for i, test := range parseReplicationDecisionTest {
+		dsc, err := parseReplicateDecision(test.expDsc.String())
+
+		if err != nil {
+			if test.expErr != err {
+				t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr)
+			}
+			continue
+		}
+		if len(dsc.targetsMap) != len(test.expDsc.targetsMap) {
+			t.Errorf("Test%d (%s): Invalid number of entries in targetsMap  got %d , want %d", i+1, test.name, len(dsc.targetsMap), len(test.expDsc.targetsMap))
+		}
+		for arn, tdsc := range dsc.targetsMap {
+			expDsc, ok := test.expDsc.targetsMap[arn]
+			if !ok || expDsc != tdsc {
+				t.Errorf("Test%d (%s): Invalid  target replicate decision: got %+v, want %+v", i+1, test.name, tdsc, expDsc)
+			}
+		}
+	}
+}
+
+var replicationStateTest = []struct {
+	name      string
+	rs        ReplicationState
+	arn       string
+	expStatus replication.StatusType
+}{
+	{ // 1. no replication status header
+		name:      "no replicated targets",
+		rs:        ReplicationState{},
+		expStatus: replication.StatusType(""),
+	},
+	{ // 2. replication status for one target
+		name:      "replication status for one target",
+		rs:        ReplicationState{ReplicationStatusInternal: "arn1=PENDING;", Targets: map[string]replication.StatusType{"arn1": "PENDING"}},
+		expStatus: replication.Pending,
+	},
+	{ // 3. replication status for one target - incorrect format
+		name:      "replication status for one target",
+		rs:        ReplicationState{ReplicationStatusInternal: "arn1=PENDING"},
+		expStatus: replication.StatusType(""),
+	},
+	{ // 4. replication status for 3 targets, one of them failed
+		name: "replication status for 3 targets - one failed",
+		rs: ReplicationState{
+			ReplicationStatusInternal: "arn1=COMPLETED;arn2=COMPLETED;arn3=FAILED;",
+			Targets:                   map[string]replication.StatusType{"arn1": "COMPLETED", "arn2": "COMPLETED", "arn3": "FAILED"},
+		},
+		expStatus: replication.Failed,
+	},
+	{ // 5. replication status for replica version
+		name:      "replication status for replica version",
+		rs:        ReplicationState{ReplicationStatusInternal: string(replication.Replica)},
+		expStatus: replication.Replica,
+	},
+}
+
+func TestCompositeReplicationStatus(t *testing.T) {
+	for i, test := range replicationStateTest {
+		if rstatus := test.rs.CompositeReplicationStatus(); rstatus != test.expStatus {
+			t.Errorf("Test%d (%s): Overall replication status  got %s , want %s", i+1, test.name, rstatus, test.expStatus)
+		}
+	}
+}

cmd/bucket-replication_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -50,6 +51,8 @@ var replicationConfigTests = []struct {
 	info         ObjectInfo
 	name         string
 	rcfg         replicationConfig
+	dsc          ReplicateDecision
+	tgtStatuses  map[string]replication.StatusType
 	expectedSync bool
 }{
 	{ // 1. no replication config
@@ -84,8 +87,8 @@ var replicationConfigTests = []struct {
 func TestReplicationResync(t *testing.T) {
 	ctx := context.Background()
 	for i, test := range replicationConfigTests {
-		if sync := test.rcfg.Resync(ctx, test.info); sync != test.expectedSync {
+		if sync := test.rcfg.Resync(ctx, test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
-			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync, test.expectedSync)
+			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
 }
@@ -94,27 +97,37 @@ var start = UTCNow().AddDate(0, 0, -1)
 var replicationConfigTests2 = []struct {
 	info         ObjectInfo
 	name         string
-	replicate    bool
 	rcfg         replicationConfig
+	dsc          ReplicateDecision
+	tgtStatuses  map[string]replication.StatusType
 	expectedSync bool
 }{
 	{ // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies
 		// 1: Pending replication
 		name: "existing object replication on object in Pending replication status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:PENDING;",
 			ReplicationStatus:         replication.Pending,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
 		expectedSync: true,
 	},
+
 	{ // 2. replication status Failed
 		name: "existing object replication on object in Failed replication status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:FAILED",
 			ReplicationStatus:         replication.Failed,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
 		expectedSync: true,
 	},
 	{ // 3. replication status unset
@@ -123,87 +136,136 @@ var replicationConfigTests2 = []struct {
 			ReplicationStatus: replication.StatusType(""),
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
 		expectedSync: true,
 	},
 	{ // 4. replication status Complete
 		name: "existing object replication on object in Completed replication status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:COMPLETED",
 			ReplicationStatus:         replication.Completed,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", false, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
 		expectedSync: false,
 	},
 	{ // 5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present
 		name: "existing object replication with reset in progress and object in Pending status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:PENDING;",
 			ReplicationStatus:         replication.Pending,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
 	},
 	{ // 6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present
 		name: "existing object replication with reset in progress and object in Failed status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:FAILED;",
 			ReplicationStatus:         replication.Failed,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+		},
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
 	},
 	{ // 7. existing object replication enabled, versioning enabled, replication status unset & reset ID present
 		name: "existing object replication with reset in progress and object never replicated before",
 		info: ObjectInfo{Size: 100,
 			ReplicationStatus: replication.StatusType(""),
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
 		},
-		replicate:    true,
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
+
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
 	},
+
 	{ // 8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present
 		name: "existing object replication enabled - reset in progress for an object in Completed status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:COMPLETED;",
 			ReplicationStatus:         replication.Completed,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df8",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
 	},
 	{ // 9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different
 		name: "existing object replication enabled, newer reset in progress on object in Pending replication status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:PENDING;",
+
 			ReplicationStatus: replication.Pending,
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), "xyz")},
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().AddDate(0, 0, -1).Format(http.TimeFormat), "abc")},
-			ModTime:           UTCNow().AddDate(0, 0, -1),
+			ModTime:           UTCNow().AddDate(0, 0, -2),
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "abc", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
 	},
 	{ // 10. existing object replication enabled, versioning enabled, replication status Complete & reset done
 		name: "reset done on object in Completed Status - ineligbile for re-replication",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:COMPLETED;",
 			ReplicationStatus:         replication.Completed,
 			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
 		},
-		replicate:    true,
 		expectedSync: false,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: start,
+		}}},
+		},
 	},
 }
 
 func TestReplicationResyncwrapper(t *testing.T) {
 	for i, test := range replicationConfigTests2 {
-		if sync := test.rcfg.resync(test.info, test.replicate); sync != test.expectedSync {
+		if sync := test.rcfg.resync(test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
-			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync, test.expectedSync)
+			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
 }

cmd/bucket-stats.go

@@ -17,8 +17,40 @@
 
 package cmd
 
+import (
+	"time"
+)
+
 //go:generate msgp -file $GOFILE
 
+// ReplicationLatency holds information of bucket operations latency, such us uploads
+type ReplicationLatency struct {
+	// Single & Multipart PUTs latency
+	UploadHistogram LastMinuteLatencies
+}
+
+// Merge two replication latency into a new one
+func (rl ReplicationLatency) merge(other ReplicationLatency) (newReplLatency ReplicationLatency) {
+	newReplLatency.UploadHistogram = rl.UploadHistogram.Merge(other.UploadHistogram)
+	return
+}
+
+// Get upload latency of each object size range
+func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) {
+	ret = make(map[string]uint64)
+	avg := rl.UploadHistogram.GetAvg()
+	for k, v := range avg {
+		// Convert nanoseconds to milliseconds
+		ret[sizeTagToString(k)] = v.avg() / uint64(time.Millisecond)
+	}
+	return
+}
+
+// Update replication upload latency with a new value
+func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
+	rl.UploadHistogram.Add(size, duration)
+}
+
 // BucketStats bucket statistics
 type BucketStats struct {
 	ReplicationStats BucketReplicationStats
@@ -27,16 +59,67 @@ type BucketStats struct {
 // BucketReplicationStats represents inline replication statistics
 // such as pending, failed and completed bytes in total for a bucket
 type BucketReplicationStats struct {
+	Stats map[string]*BucketReplicationStat
 	// Pending size in bytes
-	PendingSize uint64 `json:"pendingReplicationSize"`
+	PendingSize int64 `json:"pendingReplicationSize"`
 	// Completed size in bytes
-	ReplicatedSize uint64 `json:"completedReplicationSize"`
+	ReplicatedSize int64 `json:"completedReplicationSize"`
 	// Total Replica size in bytes
-	ReplicaSize uint64 `json:"replicaSize"`
+	ReplicaSize int64 `json:"replicaSize"`
 	// Failed size in bytes
-	FailedSize uint64 `json:"failedReplicationSize"`
+	FailedSize int64 `json:"failedReplicationSize"`
 	// Total number of pending operations including metadata updates
-	PendingCount uint64 `json:"pendingReplicationCount"`
+	PendingCount int64 `json:"pendingReplicationCount"`
 	// Total number of failed operations including metadata updates
-	FailedCount uint64 `json:"failedReplicationCount"`
+	FailedCount int64 `json:"failedReplicationCount"`
+}
+
+// Empty returns true if there are no target stats
+func (brs *BucketReplicationStats) Empty() bool {
+	return len(brs.Stats) == 0 && brs.ReplicaSize == 0
+}
+
+// Clone creates a new BucketReplicationStats copy
+func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
+	// This is called only by replicationStats cache and already holds a
+	// read lock before calling Clone()
+
+	c = brs
+	// We need to copy the map, so we do not reference the one in `brs`.
+	c.Stats = make(map[string]*BucketReplicationStat, len(brs.Stats))
+	for arn, st := range brs.Stats {
+		// make a copy of `*st`
+		s := *st
+		c.Stats[arn] = &s
+	}
+	return c
+}
+
+// BucketReplicationStat represents inline replication statistics
+// such as pending, failed and completed bytes in total for a bucket
+// remote target
+type BucketReplicationStat struct {
+	// Pending size in bytes
+	PendingSize int64 `json:"pendingReplicationSize"`
+	// Completed size in bytes
+	ReplicatedSize int64 `json:"completedReplicationSize"`
+	// Total Replica size in bytes
+	ReplicaSize int64 `json:"replicaSize"`
+	// Failed size in bytes
+	FailedSize int64 `json:"failedReplicationSize"`
+	// Total number of pending operations including metadata updates
+	PendingCount int64 `json:"pendingReplicationCount"`
+	// Total number of failed operations including metadata updates
+	FailedCount int64 `json:"failedReplicationCount"`
+	// Replication latency information
+	Latency ReplicationLatency `json:"replicationLatency"`
+}
+
+func (bs *BucketReplicationStat) hasReplicationUsage() bool {
+	return bs.FailedSize > 0 ||
+		bs.ReplicatedSize > 0 ||
+		bs.ReplicaSize > 0 ||
+		bs.FailedCount > 0 ||
+		bs.PendingCount > 0 ||
+		bs.PendingSize > 0
 }

cmd/bucket-stats_gen.go

@@ -6,6 +6,318 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+// DecodeMsg implements msgp.Decodable
+func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "PendingSize":
+			z.PendingSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "PendingSize")
+				return
+			}
+		case "ReplicatedSize":
+			z.ReplicatedSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicatedSize")
+				return
+			}
+		case "ReplicaSize":
+			z.ReplicaSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicaSize")
+				return
+			}
+		case "FailedSize":
+			z.FailedSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "FailedSize")
+				return
+			}
+		case "PendingCount":
+			z.PendingCount, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "PendingCount")
+				return
+			}
+		case "FailedCount":
+			z.FailedCount, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "FailedCount")
+				return
+			}
+		case "Latency":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Latency")
+				return
+			}
+			for zb0002 > 0 {
+				zb0002--
+				field, err = dc.ReadMapKeyPtr()
+				if err != nil {
+					err = msgp.WrapError(err, "Latency")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "UploadHistogram":
+					err = z.Latency.UploadHistogram.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Latency", "UploadHistogram")
+						return
+					}
+				default:
+					err = dc.Skip()
+					if err != nil {
+						err = msgp.WrapError(err, "Latency")
+						return
+					}
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 7
+	// write "PendingSize"
+	err = en.Append(0x87, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.PendingSize)
+	if err != nil {
+		err = msgp.WrapError(err, "PendingSize")
+		return
+	}
+	// write "ReplicatedSize"
+	err = en.Append(0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.ReplicatedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	// write "ReplicaSize"
+	err = en.Append(0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.ReplicaSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
+	// write "FailedSize"
+	err = en.Append(0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.FailedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "FailedSize")
+		return
+	}
+	// write "PendingCount"
+	err = en.Append(0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.PendingCount)
+	if err != nil {
+		err = msgp.WrapError(err, "PendingCount")
+		return
+	}
+	// write "FailedCount"
+	err = en.Append(0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.FailedCount)
+	if err != nil {
+		err = msgp.WrapError(err, "FailedCount")
+		return
+	}
+	// write "Latency"
+	err = en.Append(0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79)
+	if err != nil {
+		return
+	}
+	// map header, size 1
+	// write "UploadHistogram"
+	err = en.Append(0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d)
+	if err != nil {
+		return
+	}
+	err = z.Latency.UploadHistogram.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Latency", "UploadHistogram")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 7
+	// string "PendingSize"
+	o = append(o, 0x87, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.PendingSize)
+	// string "ReplicatedSize"
+	o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.ReplicatedSize)
+	// string "ReplicaSize"
+	o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.ReplicaSize)
+	// string "FailedSize"
+	o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.FailedSize)
+	// string "PendingCount"
+	o = append(o, 0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	o = msgp.AppendInt64(o, z.PendingCount)
+	// string "FailedCount"
+	o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	o = msgp.AppendInt64(o, z.FailedCount)
+	// string "Latency"
+	o = append(o, 0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79)
+	// map header, size 1
+	// string "UploadHistogram"
+	o = append(o, 0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d)
+	o, err = z.Latency.UploadHistogram.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Latency", "UploadHistogram")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "PendingSize":
+			z.PendingSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PendingSize")
+				return
+			}
+		case "ReplicatedSize":
+			z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicatedSize")
+				return
+			}
+		case "ReplicaSize":
+			z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicaSize")
+				return
+			}
+		case "FailedSize":
+			z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "FailedSize")
+				return
+			}
+		case "PendingCount":
+			z.PendingCount, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PendingCount")
+				return
+			}
+		case "FailedCount":
+			z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "FailedCount")
+				return
+			}
+		case "Latency":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Latency")
+				return
+			}
+			for zb0002 > 0 {
+				zb0002--
+				field, bts, err = msgp.ReadMapKeyZC(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Latency")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "UploadHistogram":
+					bts, err = z.Latency.UploadHistogram.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Latency", "UploadHistogram")
+						return
+					}
+				default:
+					bts, err = msgp.Skip(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Latency")
+						return
+					}
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BucketReplicationStat) Msgsize() (s int) {
+	s = 1 + 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + 8 + 1 + 16 + z.Latency.UploadHistogram.Msgsize()
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -24,38 +336,80 @@ func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Stats")
+				return
+			}
+			if z.Stats == nil {
+				z.Stats = make(map[string]*BucketReplicationStat, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 *BucketReplicationStat
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				if dc.IsNil() {
+					err = dc.ReadNil()
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+					za0002 = nil
+				} else {
+					if za0002 == nil {
+						za0002 = new(BucketReplicationStat)
+					}
+					err = za0002.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
 		case "PendingSize":
-			z.PendingSize, err = dc.ReadUint64()
+			z.PendingSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "PendingSize")
 				return
 			}
 		case "ReplicatedSize":
-			z.ReplicatedSize, err = dc.ReadUint64()
+			z.ReplicatedSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicatedSize")
 				return
 			}
 		case "ReplicaSize":
-			z.ReplicaSize, err = dc.ReadUint64()
+			z.ReplicaSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicaSize")
 				return
 			}
 		case "FailedSize":
-			z.FailedSize, err = dc.ReadUint64()
+			z.FailedSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "FailedSize")
 				return
 			}
 		case "PendingCount":
-			z.PendingCount, err = dc.ReadUint64()
+			z.PendingCount, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "PendingCount")
 				return
 			}
 		case "FailedCount":
-			z.FailedCount, err = dc.ReadUint64()
+			z.FailedCount, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "FailedCount")
 				return
@@ -73,13 +427,42 @@ func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 7
-	// write "PendingSize"
+	// write "Stats"
-	err = en.Append(0x86, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	err = en.Append(0x87, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.PendingSize)
+	err = en.WriteMapHeader(uint32(len(z.Stats)))
+	if err != nil {
+		err = msgp.WrapError(err, "Stats")
+		return
+	}
+	for za0001, za0002 := range z.Stats {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Stats")
+			return
+		}
+		if za0002 == nil {
+			err = en.WriteNil()
+			if err != nil {
+				return
+			}
+		} else {
+			err = za0002.EncodeMsg(en)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats", za0001)
+				return
+			}
+		}
+	}
+	// write "PendingSize"
+	err = en.Append(0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.PendingSize)
 	if err != nil {
 		err = msgp.WrapError(err, "PendingSize")
 		return
@@ -89,7 +472,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.ReplicatedSize)
+	err = en.WriteInt64(z.ReplicatedSize)
 	if err != nil {
 		err = msgp.WrapError(err, "ReplicatedSize")
 		return
@@ -99,7 +482,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.ReplicaSize)
+	err = en.WriteInt64(z.ReplicaSize)
 	if err != nil {
 		err = msgp.WrapError(err, "ReplicaSize")
 		return
@@ -109,7 +492,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.FailedSize)
+	err = en.WriteInt64(z.FailedSize)
 	if err != nil {
 		err = msgp.WrapError(err, "FailedSize")
 		return
@@ -119,7 +502,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.PendingCount)
+	err = en.WriteInt64(z.PendingCount)
 	if err != nil {
 		err = msgp.WrapError(err, "PendingCount")
 		return
@@ -129,7 +512,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.FailedCount)
+	err = en.WriteInt64(z.FailedCount)
 	if err != nil {
 		err = msgp.WrapError(err, "FailedCount")
 		return
@@ -140,25 +523,40 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketReplicationStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
+	// string "Stats"
+	o = append(o, 0x87, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Stats)))
+	for za0001, za0002 := range z.Stats {
+		o = msgp.AppendString(o, za0001)
+		if za0002 == nil {
+			o = msgp.AppendNil(o)
+		} else {
+			o, err = za0002.MarshalMsg(o)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats", za0001)
+				return
+			}
+		}
+	}
 	// string "PendingSize"
-	o = append(o, 0x86, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	o = append(o, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.PendingSize)
+	o = msgp.AppendInt64(o, z.PendingSize)
 	// string "ReplicatedSize"
 	o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.ReplicatedSize)
+	o = msgp.AppendInt64(o, z.ReplicatedSize)
 	// string "ReplicaSize"
 	o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.ReplicaSize)
+	o = msgp.AppendInt64(o, z.ReplicaSize)
 	// string "FailedSize"
 	o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.FailedSize)
+	o = msgp.AppendInt64(o, z.FailedSize)
 	// string "PendingCount"
 	o = append(o, 0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74)
-	o = msgp.AppendUint64(o, z.PendingCount)
+	o = msgp.AppendInt64(o, z.PendingCount)
 	// string "FailedCount"
 	o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74)
-	o = msgp.AppendUint64(o, z.FailedCount)
+	o = msgp.AppendInt64(o, z.FailedCount)
 	return
 }
 
@@ -180,38 +578,79 @@ func (z *BucketReplicationStats) UnmarshalMsg(bts []byte) (o []byte, err error)
 			return
 		}
 		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats")
+				return
+			}
+			if z.Stats == nil {
+				z.Stats = make(map[string]*BucketReplicationStat, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 *BucketReplicationStat
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				if msgp.IsNil(bts) {
+					bts, err = msgp.ReadNilBytes(bts)
+					if err != nil {
+						return
+					}
+					za0002 = nil
+				} else {
+					if za0002 == nil {
+						za0002 = new(BucketReplicationStat)
+					}
+					bts, err = za0002.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
 		case "PendingSize":
-			z.PendingSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.PendingSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "PendingSize")
 				return
 			}
 		case "ReplicatedSize":
-			z.ReplicatedSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicatedSize")
 				return
 			}
 		case "ReplicaSize":
-			z.ReplicaSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicaSize")
 				return
 			}
 		case "FailedSize":
-			z.FailedSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "FailedSize")
 				return
 			}
 		case "PendingCount":
-			z.PendingCount, bts, err = msgp.ReadUint64Bytes(bts)
+			z.PendingCount, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "PendingCount")
 				return
 			}
 		case "FailedCount":
-			z.FailedCount, bts, err = msgp.ReadUint64Bytes(bts)
+			z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "FailedCount")
 				return
@@ -230,7 +669,19 @@ func (z *BucketReplicationStats) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketReplicationStats) Msgsize() (s int) {
-	s = 1 + 12 + msgp.Uint64Size + 15 + msgp.Uint64Size + 12 + msgp.Uint64Size + 11 + msgp.Uint64Size + 13 + msgp.Uint64Size + 12 + msgp.Uint64Size
+	s = 1 + 6 + msgp.MapHeaderSize
+	if z.Stats != nil {
+		for za0001, za0002 := range z.Stats {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001)
+			if za0002 == nil {
+				s += msgp.NilSize
+			} else {
+				s += za0002.Msgsize()
+			}
+		}
+	}
+	s += 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size
 	return
 }
 
@@ -340,3 +791,110 @@ func (z *BucketStats) Msgsize() (s int) {
 	s = 1 + 17 + z.ReplicationStats.Msgsize()
 	return
 }
+
+// DecodeMsg implements msgp.Decodable
+func (z *ReplicationLatency) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UploadHistogram":
+			err = z.UploadHistogram.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "UploadHistogram")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *ReplicationLatency) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "UploadHistogram"
+	err = en.Append(0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d)
+	if err != nil {
+		return
+	}
+	err = z.UploadHistogram.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "UploadHistogram")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *ReplicationLatency) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "UploadHistogram"
+	o = append(o, 0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d)
+	o, err = z.UploadHistogram.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "UploadHistogram")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ReplicationLatency) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UploadHistogram":
+			bts, err = z.UploadHistogram.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "UploadHistogram")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *ReplicationLatency) Msgsize() (s int) {
+	s = 1 + 16 + z.UploadHistogram.Msgsize()
+	return
+}

cmd/bucket-stats_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalBucketReplicationStat(t *testing.T) {
+	v := BucketReplicationStat{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketReplicationStat(t *testing.T) {
+	v := BucketReplicationStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketReplicationStat Msgsize() is inaccurate")
+	}
+
+	vn := BucketReplicationStat{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalBucketReplicationStats(t *testing.T) {
 	v := BucketReplicationStats{}
 	bts, err := v.MarshalMsg(nil)
@@ -234,3 +347,116 @@ func BenchmarkDecodeBucketStats(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalReplicationLatency(t *testing.T) {
+	v := ReplicationLatency{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationLatency(b *testing.B) {
+	v := ReplicationLatency{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationLatency(b *testing.B) {
+	v := ReplicationLatency{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationLatency(b *testing.B) {
+	v := ReplicationLatency{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationLatency(t *testing.T) {
+	v := ReplicationLatency{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationLatency Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationLatency{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationLatency(b *testing.B) {
+	v := ReplicationLatency{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationLatency(b *testing.B) {
+	v := ReplicationLatency{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-targets.go

@@ -19,8 +19,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"net/http"
 	"sync"
 	"time"
@@ -30,6 +28,7 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/kms"
@@ -37,7 +36,7 @@ import (
 )
 
 const (
-	defaultHealthCheckDuration = 100 * time.Second
+	defaultHealthCheckDuration = 30 * time.Second
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -93,6 +92,9 @@ func (sys *BucketTargetSys) Delete(bucket string) {
 		return
 	}
 	for _, t := range tgts {
+		if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
+			tgt.healthCancelFn()
+		}
 		delete(sys.arnRemotesMap, t.Arn)
 	}
 	delete(sys.targetsMap, bucket)
@@ -136,24 +138,28 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	defer sys.Unlock()
 
 	tgts := sys.targetsMap[bucket]
-
 	newtgts := make([]madmin.BucketTarget, len(tgts))
 	found := false
 	for idx, t := range tgts {
 		if t.Type == tgt.Type {
-			if t.Arn == tgt.Arn && !update {
+			if t.Arn == tgt.Arn {
+				if !update {
 					return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
 				}
 				newtgts[idx] = *tgt
 				found = true
 				continue
 			}
+		}
 		newtgts[idx] = t
 	}
 	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
-
+	// cancel health check for previous target client to avoid leak.
+	if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
+		prevClnt.healthCancelFn()
+	}
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
 	sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
@@ -194,7 +200,9 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	if arn.Type == madmin.ReplicationService {
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
-		if err == nil && rcfg.RoleArn == arnStr {
+		if err == nil {
+			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{}) {
+				if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) {
 					sys.RLock()
 					_, ok := sys.arnRemotesMap[arnStr]
 					sys.RUnlock()
@@ -203,6 +211,8 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 					}
 				}
 			}
+		}
+	}
 
 	// delete ARN type from list of matching targets
 	sys.Lock()
@@ -224,6 +234,9 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	sys.targetsMap[bucket] = targets
+	if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
+		tgt.healthCancelFn()
+	}
 	delete(sys.arnRemotesMap, arnStr)
 	sys.updateBandwidthLimit(bucket, 0)
 	return nil
@@ -259,22 +272,6 @@ func NewBucketTargetSys() *BucketTargetSys {
 	}
 }
 
-// Init initializes the bucket targets subsystem for buckets which have targets configured.
-func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, bucket targets is not supported.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load bucket targets once during boot in background.
-	go sys.load(ctx, buckets, objAPI)
-	return nil
-}
-
 // UpdateAllTargets updates target to reflect metadata updates
 func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
 	if sys == nil {
@@ -286,6 +283,9 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 		// remove target and arn association
 		if tgts, ok := sys.targetsMap[bucket]; ok {
 			for _, t := range tgts {
+				if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
+					tgt.healthCancelFn()
+				}
 				delete(sys.arnRemotesMap, t.Arn)
 			}
 		}
@@ -309,23 +309,20 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 }
 
 // create minio-go clients for buckets having remote targets
-func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
-	for _, bucket := range buckets {
+	cfg := meta.bucketTargetConfig
-		cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket.Name)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
 	if cfg == nil || cfg.Empty() {
-			continue
+		return
 	}
+	sys.Lock()
+	defer sys.Unlock()
 	if len(cfg.Targets) > 0 {
 		sys.targetsMap[bucket.Name] = cfg.Targets
 	}
 	for _, tgt := range cfg.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
-				logger.LogIf(ctx, err)
+			logger.LogIf(GlobalContext, err)
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = tgtClient
@@ -333,7 +330,6 @@ func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objA
 	}
 	sys.targetsMap[bucket.Name] = cfg.Targets
 }
-}
 
 // getRemoteTargetInstanceTransport contains a singleton roundtripper.
 var getRemoteTargetInstanceTransport http.RoundTripper
@@ -360,6 +356,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
 	}
+	cancelFn, err := api.HealthCheck(hcDuration)
+	if err != nil {
+		return nil, err
+	}
 	tc := &TargetClient{
 		Client:              api,
 		healthCheckDuration: hcDuration,
@@ -367,6 +367,9 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 		Bucket:              tcfg.TargetBucket,
 		StorageClass:        tcfg.StorageClass,
 		disableProxy:        tcfg.DisableProxy,
+		healthCancelFn:      cancelFn,
+		ARN:                 tcfg.Arn,
+		ResetID:             tcfg.ResetID,
 	}
 	return tc, nil
 }
@@ -390,14 +393,9 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 
 // generate ARN that is unique to this target type
 func generateARN(t *madmin.BucketTarget) string {
-	hash := sha256.New()
-	hash.Write([]byte(t.Type))
-	hash.Write([]byte(t.Region))
-	hash.Write([]byte(t.TargetBucket))
-	hashSum := hex.EncodeToString(hash.Sum(nil))
 	arn := madmin.ARN{
 		Type:   t.Type,
-		ID:     hashSum,
+		ID:     mustGetUUID(),
 		Region: t.Region,
 		Bucket: t.TargetBucket,
 	}
@@ -445,4 +443,7 @@ type TargetClient struct {
 	replicateSync       bool
 	StorageClass        string // storage class on remote
 	disableProxy        bool
+	healthCancelFn      context.CancelFunc // cancellation function for client healthcheck
+	ARN                 string             // ARN to uniquely identify remote target
+	ResetID             string
 }

cmd/bucket-versioning-handler.go

@@ -63,6 +63,15 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL)
+		return
+	}
+
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",

cmd/common-main.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -34,10 +37,12 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	fcolor "github.com/fatih/color"
 	"github.com/go-openapi/loads"
+	"github.com/inconshreveable/mousetrap"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	consoleCerts "github.com/minio/console/pkg/certs"
@@ -51,7 +56,6 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/handlers"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -59,29 +63,59 @@ import (
 	"github.com/minio/pkg/ellipses"
 	"github.com/minio/pkg/env"
 	xnet "github.com/minio/pkg/net"
+	"github.com/rs/dnscache"
 )
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+var shardDiskTimeDelta time.Duration
 var defaultAWSCredProvider []credentials.Provider
 
 func init() {
+	if runtime.GOOS == "windows" {
+		if mousetrap.StartedByExplorer() {
+			fmt.Printf("Don't double-click %s\n", os.Args[0])
+			fmt.Println("You need to open cmd.exe/PowerShell and run it from the command line")
+			fmt.Println("Refer to the docs here on how to run it as a Windows Service https://github.com/minio/minio-service/tree/master/windows")
+			fmt.Println("Press the Enter Key to Exit")
+			fmt.Scanln()
+			os.Exit(1)
+		}
+	}
+
 	rand.Seed(time.Now().UTC().UnixNano())
 
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
-		// 30 seconds matches the orchestrator DNS TTLs, have
-		// a 5 second timeout to lookup from DNS servers.
-		globalDNSCache = xhttp.NewDNSCache(30*time.Second, 5*time.Second, logger.LogOnceIf)
-	} else {
-		// On bare-metals DNS do not change often, so it is
-		// safe to assume a higher timeout upto 10 minutes.
-		globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
-	}
 	initGlobalContext()
 
+	options := dnscache.ResolverRefreshOptions{
+		ClearUnused:      true,
+		PersistOnFailure: false,
+	}
+
+	// Call to refresh will refresh names in cache. If you pass true, it will also
+	// remove cached names not looked up since the last call to Refresh. It is a good idea
+	// to call this method on a regular interval.
+	go func() {
+		var t *time.Ticker
+		if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
+			t = time.NewTicker(1 * time.Minute)
+		} else {
+			t = time.NewTicker(10 * time.Minute)
+		}
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				globalDNSCache.RefreshWithOptions(options)
+			case <-GlobalContext.Done():
+				return
+			}
+		}
+	}()
+
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
@@ -104,6 +138,12 @@ func init() {
 		},
 	}
 
+	var err error
+	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
+	if err != nil {
+		shardDiskTimeDelta = 1 * time.Minute
+	}
+
 	// All minio-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
@@ -138,22 +178,33 @@ func minioConfigToConsoleFeatures() {
 	}
 	// if IDP is enabled, set IDP environment variables
 	if globalOpenIDConfig.URL != nil {
-		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.DiscoveryDoc.Issuer)
+		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.URL.String())
 		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ClientID)
 		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ClientSecret)
 		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
 		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ClientID)
 		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.DiscoveryDoc.ScopesSupported, ","))
+		if globalOpenIDConfig.ClaimUserinfo {
+			os.Setenv("CONSOLE_IDP_USERINFO", config.EnableOn)
+		}
+		if globalOpenIDConfig.RedirectURIDynamic {
+			// Enable dynamic redirect-uri's based on incoming 'host' header,
+			// Overrides any other callback URL.
+			os.Setenv("CONSOLE_IDP_CALLBACK_DYNAMIC", config.EnableOn)
+		}
 		if globalOpenIDConfig.RedirectURI != "" {
 			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.RedirectURI)
 		} else {
 			os.Setenv("CONSOLE_IDP_CALLBACK", getConsoleEndpoints()[0]+"/oauth_callback")
 		}
 	}
-	os.Setenv("CONSOLE_MINIO_REGION", globalServerRegion)
+	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
-	if globalSubnetLicense != "" {
+	if globalSubnetConfig.License != "" {
-		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetLicense)
+		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
+	}
+	if globalSubnetConfig.APIKey != "" {
+		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
 	}
 }
 
@@ -179,16 +230,16 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
-	noLog := func(string, ...interface{}) {
+	api := operations.NewConsoleAPI(swaggerSpec)
-		// nothing to log
+
-	}
+	if !serverDebugLog {
+		// Disable console logging if server debug log is not enabled
+		noLog := func(string, ...interface{}) {}
 
-	// Initialize MinIO loggers
 		restapi.LogInfo = noLog
 		restapi.LogError = noLog
-
-	api := operations.NewConsoleAPI(swaggerSpec)
 		api.Logger = noLog
+	}
 
 	server := restapi.NewServer(api)
 	// register all APIs
@@ -266,7 +317,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
+	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -392,7 +443,166 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
 }
 
+type envKV struct {
+	Key   string
+	Value string
+	Skip  bool
+}
+
+func (e envKV) String() string {
+	if e.Skip {
+		return ""
+	}
+	return fmt.Sprintf("%s=%s", e.Key, e.Value)
+}
+
+func parsEnvEntry(envEntry string) (envKV, error) {
+	envEntry = strings.TrimSpace(envEntry)
+	if envEntry == "" {
+		// Skip all empty lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+	const envSeparator = "="
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	if len(envTokens) != 2 {
+		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
+	}
+
+	key := envTokens[0]
+	val := envTokens[1]
+
+	// Remove quotes from the value if found
+	if len(val) >= 2 {
+		quote := val[0]
+		if (quote == '"' || quote == '\'') && val[len(val)-1] == quote {
+			val = val[1 : len(val)-1]
+		}
+	}
+
+	return envKV{
+		Key:   key,
+		Value: val,
+	}, nil
+}
+
+// Similar to os.Environ returns a copy of strings representing
+// the environment values from a file, in the form "key, value".
+// in a structured form.
+func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
+	f, err := os.Open(envConfigFile)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	var ekvs []envKV
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		ekv, err := parsEnvEntry(scanner.Text())
+		if err != nil {
+			return nil, err
+		}
+		if ekv.Skip {
+			// Skips empty lines
+			continue
+		}
+		ekvs = append(ekvs, ekv)
+	}
+	if err = scanner.Err(); err != nil {
+		return nil, err
+	}
+	return ekvs, nil
+}
+
+func readFromSecret(sp string) (string, error) {
+	// Supports reading path from docker secrets, filename is
+	// relative to /run/secrets/ position.
+	if isFile(pathJoin("/run/secrets/", sp)) {
+		sp = pathJoin("/run/secrets/", sp)
+	}
+	credBuf, err := ioutil.ReadFile(sp)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return "", nil
+		}
+		return "", err
+	}
+	return string(bytes.TrimSpace(credBuf)), nil
+}
+
+func loadEnvVarsFromFiles() {
+	if env.IsSet(config.EnvAccessKeyFile) {
+		accessKey, err := readFromSecret(env.Get(config.EnvAccessKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if accessKey != "" {
+			os.Setenv(config.EnvRootUser, accessKey)
+		}
+	}
+
+	if env.IsSet(config.EnvSecretKeyFile) {
+		secretKey, err := readFromSecret(env.Get(config.EnvSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if secretKey != "" {
+			os.Setenv(config.EnvRootPassword, secretKey)
+		}
+	}
+
+	if env.IsSet(config.EnvRootUserFile) {
+		rootUser, err := readFromSecret(env.Get(config.EnvRootUserFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootUser != "" {
+			os.Setenv(config.EnvRootUser, rootUser)
+		}
+	}
+
+	if env.IsSet(config.EnvRootPasswordFile) {
+		rootPassword, err := readFromSecret(env.Get(config.EnvRootPasswordFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootPassword != "" {
+			os.Setenv(config.EnvRootPassword, rootPassword)
+		}
+	}
+
+	if env.IsSet(config.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
+		}
+		if kmsSecret != "" {
+			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+		}
+	}
+
+	if env.IsSet(config.EnvConfigEnvFile) {
+		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the config environment file")
+		}
+		for _, ekv := range ekvs {
+			os.Setenv(ekv.Key, ekv.Value)
+		}
+	}
+}
+
 func handleCommonEnvVars() {
+	loadEnvVarsFromFiles()
+
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -489,11 +699,12 @@ func handleCommonEnvVars() {
 	// in-place update is off.
 	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff)
 
-	// Check if the supported credential env vars, "MINIO_ROOT_USER" and
+	// Check if the supported credential env vars,
-	// "MINIO_ROOT_PASSWORD" are provided
+	// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
 	// Warn user if deprecated environment variables,
 	// "MINIO_ACCESS_KEY" and "MINIO_SECRET_KEY", are defined
 	// Check all error conditions first
+	//nolint:gocritic
 	if !env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		logger.Fatal(config.ErrMissingEnvCredentialRootUser(nil), "Unable to start MinIO")
 	} else if env.IsSet(config.EnvRootUser) && !env.IsSet(config.EnvRootPassword) {
@@ -510,24 +721,24 @@ func handleCommonEnvVars() {
 	// are defined or both are not defined.
 	// Check both cases and authenticate them if correctly defined
 	var user, password string
-	haveRootCredentials := false
+	var hasCredentials bool
-	haveAccessCredentials := false
+	//nolint:gocritic
 	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		user = env.Get(config.EnvRootUser, "")
 		password = env.Get(config.EnvRootPassword, "")
-		haveRootCredentials = true
+		hasCredentials = true
 	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
-		haveAccessCredentials = true
+		hasCredentials = true
 	}
-	if haveRootCredentials || haveAccessCredentials {
+	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
 			logger.Fatal(config.ErrInvalidCredentials(err),
 				"Unable to validate credentials inherited from the shell environment")
 		}
-		if haveAccessCredentials {
+		if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,
@@ -595,12 +806,6 @@ func handleCommonEnvVars() {
 		}
 		GlobalKMS = KMS
 	}
-
-	if tiers := env.Get("_MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY", ""); tiers != "" {
-		globalDebugRemoteTiersImmediately = strings.Split(tiers, ",")
-	}
-
-	globalSubnetLicense = env.Get(config.EnvMinIOSubnetLicense, "")
 }
 
 func logStartupMessage(msg string) {
@@ -685,6 +890,10 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 	}
 	secureConn = true
+
+	// syscall.SIGHUP to reload the certs.
+	manager.ReloadOnSignal(syscall.SIGHUP)
+
 	return x509Certs, manager, secureConn, nil
 }
 

cmd/common-main_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"reflect"
+	"testing"
+)
+
+func Test_readFromSecret(t *testing.T) {
+	testCases := []struct {
+		content       string
+		expectedErr   bool
+		expectedValue string
+	}{
+		{
+			"value\n",
+			false,
+			"value",
+		},
+		{
+			" \t\n Hello, Gophers \n\t\r\n",
+			false,
+			"Hello, Gophers",
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			value, err := readFromSecret(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if value != testCase.expectedValue {
+				t.Errorf("Expected %s, got %s", testCase.expectedValue, value)
+			}
+		})
+	}
+}
+
+func Test_minioEnvironFromFile(t *testing.T) {
+	testCases := []struct {
+		content      string
+		expectedErr  bool
+		expectedEkvs []envKV
+	}{
+		{`
+export MINIO_ROOT_USER=minio
+export MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		// Value with double quotes
+		{`export MINIO_ROOT_USER="minio"`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		// Value with single quotes
+		{`export MINIO_ROOT_USER='minio'`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		{`
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		{`
+export MINIO_ROOT_USERminio
+export MINIO_ROOT_PASSWORD=minio123`,
+			true,
+			nil,
+		},
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			ekvs, err := minioEnvironFromFile(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) {
+				t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs)
+			}
+		})
+	}
+}

cmd/config-common.go

@@ -29,27 +29,31 @@ import (
 
 var errConfigNotFound = errors.New("config file not found")
 
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+func readConfigWithMetadata(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, ObjectInfo, error) {
-	// Read entire content by setting size to -1
 	r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
-			return nil, errConfigNotFound
+			return nil, ObjectInfo{}, errConfigNotFound
 		}
 
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	defer r.Close()
 
 	buf, err := ioutil.ReadAll(r)
 	if err != nil {
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	if len(buf) == 0 {
-		return nil, errConfigNotFound
+		return nil, ObjectInfo{}, errConfigNotFound
 	}
-	return buf, nil
+	return buf, r.ObjInfo, nil
+}
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	buf, _, err := readConfigWithMetadata(ctx, objAPI, configFile)
+	return buf, err
 }
 
 type objectDeleter interface {
@@ -57,7 +61,9 @@ type objectDeleter interface {
 }
 
 func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
-	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
+	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{
+		DeletePrefix: true,
+	})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound
 	}

cmd/config-current.go

@@ -34,10 +34,12 @@ import (
 	"github.com/minio/minio/internal/config/heal"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
 	"github.com/minio/minio/internal/config/scanner"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
@@ -54,15 +56,18 @@ func initHelp() {
 		config.CompressionSubSys:    compress.DefaultKVS,
 		config.IdentityLDAPSubSys:   xldap.DefaultKVS,
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
+		config.IdentityTLSSubSys:    xtls.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
+		config.SiteSubSys:           config.DefaultSiteKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
 		config.CredentialsSubSys:    config.DefaultCredentialKVS,
-		config.LoggerWebhookSubSys:  logger.DefaultKVS,
+		config.LoggerWebhookSubSys:  logger.DefaultLoggerWebhookKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
 		config.ScannerSubSys:        scanner.DefaultKVS,
+		config.SubnetSubSys:         subnet.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -75,8 +80,8 @@ func initHelp() {
 	// Captures help for each sub-system
 	var helpSubSys = config.HelpKVS{
 		config.HelpKV{
-			Key:         config.RegionSubSys,
+			Key:         config.SiteSubSys,
-			Description: "label the location of the server",
+			Description: "label the server and its location",
 		},
 		config.HelpKV{
 			Key:         config.CacheSubSys,
@@ -98,6 +103,10 @@ func initHelp() {
 			Key:         config.IdentityLDAPSubSys,
 			Description: "enable LDAP SSO support",
 		},
+		config.HelpKV{
+			Key:         config.IdentityTLSSubSys,
+			Description: "enable X.509 TLS certificate SSO support",
+		},
 		config.HelpKV{
 			Key:         config.PolicyOPASubSys,
 			Description: "[DEPRECATED] enable external OPA for policy enforcement",
@@ -179,6 +188,12 @@ func initHelp() {
 			Description:     "publish bucket notifications to Redis datastores",
 			MultipleTargets: true,
 		},
+		config.HelpKV{
+			Key:         config.SubnetSubSys,
+			Type:        "string",
+			Description: "set subnet config for the cluster e.g. api key",
+			Optional:    true,
+		},
 	}
 
 	if globalIsErasure {
@@ -192,6 +207,7 @@ func initHelp() {
 
 	var helpMap = map[string]config.HelpKVS{
 		"":                          helpSubSys, // Help for all sub-systems.
+		config.SiteSubSys:           config.SiteHelp,
 		config.RegionSubSys:         config.RegionHelp,
 		config.APISubSys:            api.Help,
 		config.StorageClassSubSys:   storageclass.Help,
@@ -202,6 +218,7 @@ func initHelp() {
 		config.ScannerSubSys:        scanner.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
+		config.IdentityTLSSubSys:    xtls.Help,
 		config.PolicyOPASubSys:      opa.Help,
 		config.LoggerWebhookSubSys:  logger.Help,
 		config.AuditWebhookSubSys:   logger.HelpWebhook,
@@ -216,9 +233,20 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
+		config.SubnetSubSys:         subnet.HelpSubnet,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
+
+	// save top-level help for deprecated sub-systems in a separate map.
+	deprecatedHelpKVMap := map[string]config.HelpKV{
+		config.RegionSubSys: {
+			Key:         config.RegionSubSys,
+			Description: "[DEPRECATED - use `site` instead] label the location of the server",
+		},
+	}
+
+	config.RegisterHelpDeprecatedSubSys(deprecatedHelpKVMap)
 }
 
 var (
@@ -227,7 +255,9 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
 
-func validateConfig(s config.Config, setDriveCounts []int) error {
+func validateConfig(s config.Config) error {
+	objAPI := newObjectLayerFn()
+
 	// We must have a global lock for this so nobody else modifies env while we do.
 	defer env.LockSetEnv()()
 
@@ -241,7 +271,7 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 		return err
 	}
 
-	if _, err := config.LookupRegion(s[config.RegionSubSys][config.Default]); err != nil {
+	if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -250,7 +280,10 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 	}
 
 	if globalIsErasure {
-		for _, setDriveCount := range setDriveCounts {
+		if objAPI == nil {
+			return errServerNotInitialized
+		}
+		for _, setDriveCount := range objAPI.SetDriveCounts() {
 			if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
 				return err
 			}
@@ -265,7 +298,7 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 	if err != nil {
 		return err
 	}
-	objAPI := newObjectLayerFn()
+
 	if objAPI != nil {
 		if compCfg.Enabled && !objAPI.IsCompressionSupported() {
 			return fmt.Errorf("Backend does not support compression")
@@ -294,7 +327,7 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 		}
 	}
 	if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 		return err
 	}
 
@@ -312,6 +345,12 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 			conn.Close()
 		}
 	}
+	{
+		_, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
+		if err != nil {
+			return err
+		}
+	}
 
 	if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
@@ -325,7 +364,7 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
 }
 
-func lookupConfigs(s config.Config, setDriveCounts []int) {
+func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	ctx := GlobalContext
 
 	var err error
@@ -337,7 +376,15 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		}
 	}
 
-	if dnsURL, dnsUser, dnsPass, ok := env.LookupEnv(config.EnvDNSWebhook); ok {
+	dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
+	if err != nil {
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+		}
+	}
+	if err == nil && dnsURL != "" {
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
@@ -402,9 +449,9 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
+	globalSite, err = config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -412,14 +459,13 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	globalAPIConfig.init(apiConfig, setDriveCounts)
-
 	// Initialize remote instance transport once.
 	getRemoteInstanceTransportOnce.Do(func() {
 		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
 	})
 
-	if globalIsErasure {
+	if globalIsErasure && objAPI != nil {
+		setDriveCounts := objAPI.SetDriveCounts()
 		for i, setDriveCount := range setDriveCounts {
 			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 			if err != nil {
@@ -457,8 +503,17 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
 	}
 
+	globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
+	}
+
+	if globalSTSTLSConfig.InsecureSkipVerify {
+		logger.Info("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify)
+	}
+
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody)
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
@@ -478,12 +533,17 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
 	}
 
+	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
+	}
+
 	// Load logger targets based on user's configuration
 	loggerUserAgent := getUserAgent(getMinioMode())
 
 	loggerCfg, err := logger.LookupConfig(s)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger/audit targets: %w", err))
 	}
 
 	for _, l := range loggerCfg.HTTP {
@@ -493,7 +553,7 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http logging
 			if err = logger.AddTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server logger HTTP target: %w", err))
 			}
 		}
 	}
@@ -505,7 +565,7 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http audit logging
 			if err = logger.AddAuditTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit HTTP target: %w", err))
 			}
 		}
 	}
@@ -515,7 +575,7 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 			l.LogOnce = logger.LogOnceIf
 			// Enable Kafka audit logging
 			if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit Kafka target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit Kafka target: %w", err))
 			}
 		}
 	}
@@ -531,16 +591,18 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 	}
 
 	// Apply dynamic config values
-	logger.LogIf(ctx, applyDynamicConfig(ctx, newObjectLayerFn(), s))
+	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize dynamic configuration")
+		} else {
+			logger.LogIf(ctx, err)
+		}
+	}
 }
 
 // applyDynamicConfig will apply dynamic config values.
 // Dynamic systems should be in config.SubSystemsDynamic as well.
 func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
-	if objAPI == nil {
-		return nil
-	}
-
 	// Read all dynamic configs.
 	// API
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -555,9 +617,11 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config
 	}
 
 	// Validate if the object layer supports compression.
+	if objAPI != nil {
 		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
 			return fmt.Errorf("Backend does not support compression")
 		}
+	}
 
 	// Heal
 	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
@@ -573,15 +637,17 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config
 
 	// Apply configurations.
 	// We should not fail after this.
-	globalAPIConfig.init(apiConfig, objAPI.SetDriveCounts())
+	var setDriveCounts []int
+	if objAPI != nil {
+		setDriveCounts = objAPI.SetDriveCounts()
+	}
+	globalAPIConfig.init(apiConfig, setDriveCounts)
 
 	globalCompressConfigMu.Lock()
 	globalCompressConfig = cmpCfg
 	globalCompressConfigMu.Unlock()
 
-	globalHealConfigMu.Lock()
+	globalHealConfig.Update(healCfg)
-	globalHealConfig = healCfg
-	globalHealConfigMu.Unlock()
 
 	// update dynamic scanner values.
 	scannerCycle.Update(scannerCfg.Cycle)
@@ -619,9 +685,12 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 	subSys = subSystemValue[0]
 
 	subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys)
+	if !ok {
+		subSysHelp, ok = config.HelpDeprecatedSubSysMap[subSys]
 		if !ok {
 			return Help{}, config.Errorf("unknown sub-system %s", subSys)
 		}
+	}
 
 	h, ok := config.HelpSubSysMap[subSys]
 	if !ok {
@@ -642,9 +711,7 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 		// to list the ENV, for regular k/v EnableKey is
 		// implicit, for ENVs we cannot make it implicit.
 		if subSysHelp.MultipleTargets {
-			envK := config.EnvPrefix + strings.Join([]string{
+			envK := config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(madmin.EnableKey)
-				strings.ToTitle(subSys), strings.ToTitle(madmin.EnableKey),
-			}, config.EnvWordDelimiter)
 			envHelp = append(envHelp, config.HelpKV{
 				Key:         envK,
 				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
@@ -653,9 +720,7 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 			})
 		}
 		for _, hkv := range h {
-			envK := config.EnvPrefix + strings.Join([]string{
+			envK := config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(hkv.Key)
-				strings.ToTitle(subSys), strings.ToTitle(hkv.Key),
-			}, config.EnvWordDelimiter)
 			envHelp = append(envHelp, config.HelpKV{
 				Key:         envK,
 				Description: hkv.Description,
@@ -706,7 +771,7 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg, objAPI.SetDriveCounts())
+	lookupConfigs(srvCfg, objAPI)
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config-current_test.go

@@ -36,18 +36,21 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 
-	if globalServerRegion != globalMinioDefaultRegion {
+	if globalSite.Region != globalMinioDefaultRegion {
-		t.Errorf("Expecting region `us-east-1` found %s", globalServerRegion)
+		t.Errorf("Expecting region `us-east-1` found %s", globalSite.Region)
 	}
 
 	// Set new region and verify.
 	config.SetRegion(globalServerConfig, "us-west-1")
-	region, err := config.LookupRegion(globalServerConfig[config.RegionSubSys][config.Default])
+	site, err := config.LookupSite(
+		globalServerConfig[config.SiteSubSys][config.Default],
+		globalServerConfig[config.RegionSubSys][config.Default],
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if region != "us-west-1" {
+	if site.Region != "us-west-1" {
-		t.Errorf("Expecting region `us-west-1` found %s", globalServerRegion)
+		t.Errorf("Expecting region `us-west-1` found %s", globalSite.Region)
 	}
 
 	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {

cmd/config-migrate.go

@@ -2754,7 +2754,6 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 	}
 
 	xldap.SetIdentityLDAP(newCfg, cfg.LDAPServerConfig)
-	openid.SetIdentityOpenID(newCfg, cfg.OpenID)
 	opa.SetPolicyOPAConfig(newCfg, cfg.Policy.OPA)
 	cache.SetCacheConfig(newCfg, cfg.Cache)
 	compress.SetCompressionConfig(newCfg, cfg.Compression)

cmd/config-versions.go

@@ -34,14 +34,12 @@ import (
 	"github.com/minio/pkg/quick"
 )
 
-/////////////////// Config V1 ///////////////////
 type configV1 struct {
 	Version   string `json:"version"`
 	AccessKey string `json:"accessKeyId"`
 	SecretKey string `json:"secretAccessKey"`
 }
 
-/////////////////// Config V2 ///////////////////
 type configV2 struct {
 	Version     string `json:"version"`
 	Credentials struct {
@@ -63,7 +61,6 @@ type configV2 struct {
 	} `json:"fileLogger"`
 }
 
-/////////////////// Config V3 ///////////////////
 // backendV3 type.
 type backendV3 struct {
 	Type  string   `json:"type"`

cmd/config.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"path"
 	"sort"
 	"strings"
@@ -96,7 +97,9 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 
 func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
 	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
-	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{})
+	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{
+		DeletePrefix: true,
+	})
 	return err
 }
 
@@ -155,7 +158,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	data, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			lookupConfigs(srvCfg, objAPI.SetDriveCounts())
+			lookupConfigs(srvCfg, objAPI)
 			return srvCfg, nil
 		}
 		return nil, err
@@ -166,7 +169,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 			minioMetaBucket: path.Join(minioMetaBucket, configFile),
 		})
 		if err != nil {
-			lookupConfigs(srvCfg, objAPI.SetDriveCounts())
+			lookupConfigs(srvCfg, objAPI)
 			return nil, err
 		}
 	}
@@ -183,11 +186,6 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 // ConfigSys - config system.
 type ConfigSys struct{}
 
-// Load - load config.json.
-func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
-	return sys.Init(objAPI)
-}
-
 // Init - initializes config system from config.json.
 func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {
@@ -220,18 +218,18 @@ func initConfig(objAPI ObjectLayer) error {
 	// If etcd is set then migrates /config/config.json
 	// to '<export_path>/.minio.sys/config/config.json'
 	if err := migrateConfigToMinioSys(objAPI); err != nil {
-		return err
+		return fmt.Errorf("migrateConfigToMinioSys: %w", err)
 	}
 
 	// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
 	if err := migrateMinioSysConfig(objAPI); err != nil {
-		return err
+		return fmt.Errorf("migrateMinioSysConfig: %w", err)
 	}
 
 	// Migrates backend '<export_path>/.minio.sys/config/config.json' to
 	// latest config format.
 	if err := migrateMinioSysConfigToKV(objAPI); err != nil {
-		return err
+		return fmt.Errorf("migrateMinioSysConfigToKV: %w", err)
 	}
 
 	return loadConfig(objAPI)

cmd/crossdomain-xml-handler.go

@@ -25,21 +25,13 @@ const crossDomainXML = `<?xml version="1.0"?><!DOCTYPE cross-domain-policy SYSTE
 // Standard path where an app would find cross domain policy information.
 const crossDomainXMLEntity = "/crossdomain.xml"
 
-// Cross domain policy implements http.Handler interface, implementing a custom ServerHTTP.
-type crossDomainPolicy struct {
-	handler http.Handler
-}
-
 // A cross-domain policy file is an XML document that grants a web client, such as Adobe Flash Player
 // or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains.
 // When clients request content hosted on a particular source domain and that content make requests
 // directed towards a domain other than its own, the remote domain needs to host a cross-domain
 // policy file that grants access to the source domain, allowing the client to continue the transaction.
 func setCrossDomainPolicy(h http.Handler) http.Handler {
-	return crossDomainPolicy{handler: h}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-}
-
-func (c crossDomainPolicy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		// Look for 'crossdomain.xml' in the incoming request.
 		switch r.URL.Path {
 		case crossDomainXMLEntity:
@@ -48,6 +40,6 @@ func (c crossDomainPolicy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			// Request completed, no need to serve to other handlers.
 			return
 		}
-	// Continue to serve the request further.
+		h.ServeHTTP(w, r)
-	c.handler.ServeHTTP(w, r)
+	})
 }

cmd/data-scanner.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"io/fs"
 	"math"
 	"math/rand"
 	"net/http"
@@ -29,6 +30,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
@@ -58,7 +60,6 @@ const (
 
 var (
 	globalHealConfig heal.Config
-	globalHealConfigMu sync.Mutex
 
 	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 	// Sleeper values are updated when config is loaded.
@@ -70,7 +71,17 @@ var (
 
 // initDataScanner will start the scanner in the background.
 func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
-	go runDataScanner(ctx, objAPI)
+	go func() {
+		// Run the data scanner in a loop
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				runDataScanner(ctx, objAPI)
+			}
+		}
+	}()
 }
 
 type safeDuration struct {
@@ -143,11 +154,11 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			}
 
 			// Wait before starting next cycle and wait on startup.
-			results := make(chan madmin.DataUsageInfo, 1)
+			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results)
+			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle))
 			logger.LogIf(ctx, err)
 			if err == nil {
 				// Store new cycle...
@@ -195,6 +206,22 @@ type folderScanner struct {
 	lastUpdate time.Time
 }
 
+type scannerStats struct {
+	// All fields must be accessed atomically and aligned.
+
+	accTotalObjects  uint64
+	accTotalVersions uint64
+	accFolders       uint64
+	bucketsStarted   uint64
+	bucketsFinished  uint64
+	ilmChecks        uint64
+
+	// actions records actions performed.
+	actions [lifecycle.ActionCount]uint64
+}
+
+var globalScannerStats scannerStats
+
 // Cache structure and compaction:
 //
 // A cache structure will be kept with a tree of usages.
@@ -242,6 +269,10 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 
 	logPrefix := color.Green("data-usage: ")
 	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
+	atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
+	defer func() {
+		atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
+	}()
 	if intDataUpdateTracker.debug {
 		defer func() {
 			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
@@ -320,7 +351,7 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		console.Debugf(logPrefix+"Finished scanner, %v entries (%+v) %s \n", len(s.newCache.Cache), *s.newCache.sizeRecursive(s.newCache.Info.Name), logSuffix)
 	}
 	s.newCache.Info.LastUpdate = UTCNow()
-	s.newCache.Info.NextCycle++
+	s.newCache.Info.NextCycle = cache.Info.NextCycle
 	return s.newCache, nil
 }
 
@@ -350,6 +381,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
+	atomic.AddUint64(&globalScannerStats.accFolders, 1)
 
 	for {
 		select {
@@ -406,11 +438,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
 			// Parse
 			entName = pathClean(path.Join(folder.name, entName))
-			if entName == "" {
+			if entName == "" || entName == folder.name {
 				if f.dataUsageScannerDebug {
-					console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
+					console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName)
 				}
-				return errDoneForNow
+				return nil
 			}
 			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
 			if bucket == "" {
@@ -436,7 +468,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			if typ&os.ModeDir != 0 {
 				h := hashPath(entName)
 				_, exists := f.oldCache.Cache[h.Key()]
-
+				if h == thisHash {
+					return nil
+				}
 				this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
 				delete(abandonedChildren, h.Key()) // h.Key() already accounted for.
 				if exists {
@@ -469,11 +503,14 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			item.heal = item.heal && f.healObjectSelect > 0
 
 			sz, err := f.getSize(item)
-			if err == errSkipFile {
+			if err != nil {
 				wait() // wait to proceed to next entry.
-
+				if err != errSkipFile && f.dataUsageScannerDebug {
+					console.Debugf(scannerLogPrefix+" getSize \"%v/%v\" returned err: %v\n", bucket, item.objectPath(), err)
+				}
 				return nil
 			}
+
 			// successfully read means we have a valid object.
 			foundObjects = true
 			// Remove filename i.e is the meta file to construct object name
@@ -629,6 +666,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			dirQuorum: getReadQuorum(len(f.disks)),
 			objQuorum: getReadQuorum(len(f.disks)),
 			bucket:    "",
+			strict:    false,
 		}
 
 		healObjectsPrefix := color.Green("healObjects:")
@@ -638,10 +676,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
 			}
 
+			if bucket != resolver.bucket {
+				// Bucket might be missing as well with abandoned children.
+				// make sure it is created first otherwise healing won't proceed
+				// for objects.
+				_, _ = objAPI.HealBucket(ctx, bucket, madmin.HealOpts{})
+			}
+
 			resolver.bucket = bucket
 
 			foundObjs := false
-			dangling := false
 			ctx, cancel := context.WithCancel(ctx)
 
 			err := listPathRaw(ctx, listPathRawOptions{
@@ -663,18 +707,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
 					}
 
-					// agreed value less than expected quorum
-					dangling = nAgreed < resolver.objQuorum || nAgreed < resolver.dirQuorum
-
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
-						for _, err := range errs {
+						// check if we can get one entry atleast
-							if err != nil {
+						// proceed to heal nonetheless, since
-								return
+						// this object might be dangling.
-							}
-						}
-
-						// If no errors, queue it for healing.
 						entry, _ = entries.firstFound()
 					}
 
@@ -698,9 +735,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							object:    entry.name,
 							versionID: "",
 						}, madmin.HealItemObject)
-						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 						logger.LogIf(ctx, err)
-						}
 						foundObjs = foundObjs || err == nil
 						return
 					}
@@ -715,9 +750,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							object:    fiv.Name,
 							versionID: ver.VersionID,
 						}, madmin.HealItemObject)
-						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 						logger.LogIf(ctx, err)
-						}
 						foundObjs = foundObjs || err == nil
 					}
 				},
@@ -734,30 +767,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
 
-			// If we found one or more disks with this folder, delete it.
-			if err == nil && dangling {
-				if f.dataUsageScannerDebug {
-					console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
-				}
-
-				// wait on timer per object.
-				wait := scannerSleeper.Timer(ctx)
-
-				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
-					Recursive: true,
-					Remove:    healDeleteDangling,
-				}, func(bucket, object, versionID string) error {
-					// Wait for each heal as per scanner frequency.
-					wait()
-					wait = scannerSleeper.Timer(ctx)
-					return bgSeq.queueHealTask(healSource{
-						bucket:    bucket,
-						object:    object,
-						versionID: versionID,
-					}, madmin.HealItemObject)
-				})
-			}
-
 			// Add unless healing returned an error.
 			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
@@ -820,13 +829,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 // scannerItem represents each file while walking.
 type scannerItem struct {
 	Path        string
-	Typ  os.FileMode
-
 	bucket      string // Bucket.
 	prefix      string // Only the prefix if any, does not have final object name.
 	objectName  string // Only the object name without prefixes.
-	lifeCycle   *lifecycle.Lifecycle
 	replication replicationConfig
+	lifeCycle   *lifecycle.Lifecycle
+	Typ         fs.FileMode
 	heal        bool // Has the object been selected for heal check?
 	debug       bool
 }
@@ -840,6 +848,17 @@ type sizeSummary struct {
 	replicaSize     int64
 	pendingCount    uint64
 	failedCount     uint64
+	replTargetStats map[string]replTargetSizeSummary
+	tiers           map[string]tierStats
+}
+
+// replTargetSizeSummary holds summary of replication stats by target
+type replTargetSizeSummary struct {
+	replicatedSize int64
+	pendingSize    int64
+	failedSize     int64
+	pendingCount   uint64
+	failedCount    uint64
 }
 
 type getSizeFn func(item scannerItem) (sizeSummary, error)
@@ -856,30 +875,22 @@ func (i *scannerItem) transformMetaDir() {
 	i.objectName = split[len(split)-1]
 }
 
-// actionMeta contains information used to apply actions.
-type actionMeta struct {
-	oi         ObjectInfo
-	bitRotScan bool // indicates if bitrot check was requested.
-}
-
 var applyActionsLogPrefix = color.Green("applyActions:")
+var applyVersionActionsLogPrefix = color.Green("applyVersionActions:")
 
-func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, meta actionMeta) (size int64) {
+func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi ObjectInfo) (size int64) {
 	if i.debug {
-		if meta.oi.VersionID != "" {
+		if oi.VersionID != "" {
-			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), meta.oi.VersionID)
+			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), oi.VersionID)
 		} else {
 			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
 		}
 	}
-	healOpts := madmin.HealOpts{Remove: healDeleteDangling}
+	healOpts := madmin.HealOpts{
-	if meta.bitRotScan {
+		Remove:   healDeleteDangling,
-		healOpts.ScanMode = madmin.HealDeepScan
+		ScanMode: globalHealConfig.ScanMode(),
-	}
-	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, healOpts)
-	if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-		return 0
 	}
+	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
 	if err != nil && !errors.Is(err, NotImplemented{}) {
 		logger.LogIf(ctx, err)
 		return 0
@@ -887,8 +898,8 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, meta acti
 	return res.ObjectSize
 }
 
-func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta actionMeta) (applied bool, size int64) {
+func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (applied bool, size int64) {
-	size, err := meta.oi.GetActualSize()
+	size, err := oi.GetActualSize()
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
@@ -900,21 +911,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta ac
 		return false, size
 	}
 
-	versionID := meta.oi.VersionID
+	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
+	versionID := oi.VersionID
 	action := i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
 			Name:             i.objectPath(),
-			UserTags:               meta.oi.UserTags,
+			UserTags:         oi.UserTags,
-			ModTime:                meta.oi.ModTime,
+			ModTime:          oi.ModTime,
-			VersionID:              meta.oi.VersionID,
+			VersionID:        oi.VersionID,
-			DeleteMarker:           meta.oi.DeleteMarker,
+			DeleteMarker:     oi.DeleteMarker,
-			IsLatest:               meta.oi.IsLatest,
+			IsLatest:         oi.IsLatest,
-			NumVersions:            meta.oi.NumVersions,
+			NumVersions:      oi.NumVersions,
-			SuccessorModTime:       meta.oi.SuccessorModTime,
+			SuccessorModTime: oi.SuccessorModTime,
-			RestoreOngoing:         meta.oi.RestoreOngoing,
+			RestoreOngoing:   oi.RestoreOngoing,
-			RestoreExpires:         meta.oi.RestoreExpires,
+			RestoreExpires:   oi.RestoreExpires,
-			TransitionStatus:       meta.oi.TransitionedObject.Status,
+			TransitionStatus: oi.TransitionedObject.Status,
-			RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
 		})
 	if i.debug {
 		if versionID != "" {
@@ -923,60 +934,23 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta ac
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
 		}
 	}
+	atomic.AddUint64(&globalScannerStats.actions[action], 1)
+
 	switch action {
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		return applyLifecycleAction(action, oi), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		return applyLifecycleAction(action, oi), size
 	default:
 		// No action.
-		if i.debug {
-			console.Debugf(applyActionsLogPrefix+" object not expirable: %q\n", i.objectPath())
-		}
 		return false, size
 	}
-
-	obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
-		VersionID: versionID,
-	})
-	if err != nil {
-		switch err.(type) {
-		case MethodNotAllowed: // This happens usually for a delete marker
-			if !obj.DeleteMarker { // if this is not a delete marker log and return
-				// Do nothing - heal in the future.
-				logger.LogIf(ctx, err)
-				return false, size
-			}
-		case ObjectNotFound, VersionNotFound:
-			// object not found or version not found return 0
-			return false, 0
-		default:
-			// All other errors proceed.
-			logger.LogIf(ctx, err)
-			return false, size
-		}
-	}
-
-	action = evalActionFromLifecycle(ctx, *i.lifeCycle, obj, i.debug)
-	if action != lifecycle.NoneAction {
-		applied = applyLifecycleAction(ctx, action, o, obj)
-	}
-
-	if applied {
-		switch action {
-		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			return true, size
-		}
-		// For all other lifecycle actions that remove data
-		return true, 0
-	}
-
-	return false, size
 }
 
 // applyTierObjSweep removes remote object pending deletion and the free-version
 // tracking this information.
-func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta actionMeta) {
+func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
-	if !meta.oi.TransitionedObject.FreeVersion {
+	if !oi.TransitionedObject.FreeVersion {
 		// nothing to be done
 		return
 	}
@@ -989,18 +963,18 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta
 		return err
 	}
 	// Remove the remote object
-	err := deleteObjectFromRemoteTier(ctx, meta.oi.TransitionedObject.Name, meta.oi.TransitionedObject.VersionID, meta.oi.TransitionedObject.Tier)
+	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
 
 	// Remove this free version
-	_, err = o.DeleteObject(ctx, meta.oi.Bucket, meta.oi.Name, ObjectOptions{
+	_, err = o.DeleteObject(ctx, oi.Bucket, oi.Name, ObjectOptions{
-		VersionID: meta.oi.VersionID,
+		VersionID: oi.VersionID,
 	})
 	if err == nil {
-		auditLogLifecycle(ctx, meta.oi, ILMFreeVersionDelete)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -1008,23 +982,80 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta
 
 }
 
+// applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
+// Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+	if i.lifeCycle == nil {
+		return fivs, nil
+	}
+
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
+		return fivs, nil
+	}
+
+	overflowVersions := fivs[lim+1:]
+	// current version + most recent lim noncurrent versions
+	fivs = fivs[:lim+1]
+
+	rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
+	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
+	for _, fi := range overflowVersions {
+		obj := fi.ToObjectInfo(i.bucket, i.objectPath())
+		// skip versions with object locking enabled
+		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
+			if i.debug {
+				if obj.VersionID != "" {
+					console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
+				} else {
+					console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
+				}
+			}
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
+			continue
+		}
+
+		// NoncurrentDays not passed yet.
+		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, days)) {
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
+			continue
+		}
+
+		toDel = append(toDel, ObjectToDelete{
+			ObjectName: fi.Name,
+			VersionID:  fi.VersionID,
+		})
+	}
+
+	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
+	return fivs, nil
+}
+
+// applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
+// after applying lifecycle checks configured.
+func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+}
+
 // applyActions will apply lifecycle checks on to a scanned item.
 // The resulting size on disk will always be returned.
 // The metadata will be compared to consensus on the object layer before any changes are applied.
 // If no metadata is supplied, -1 is returned if no action is taken.
-func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, meta actionMeta, sizeS *sizeSummary) int64 {
+func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) int64 {
-	i.applyTierObjSweep(ctx, o, meta)
+	applied, size := i.applyLifecycle(ctx, o, oi)
-
-	applied, size := i.applyLifecycle(ctx, o, meta)
 	// For instance, an applied lifecycle means we remove/transitioned an object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
 		if i.heal {
-			size = i.applyHealing(ctx, o, meta)
+			size = i.applyHealing(ctx, o, oi)
 		}
 		// replicate only if lifecycle rules are not applied.
-		i.healReplication(ctx, o, meta.oi.Clone(), sizeS)
+		i.healReplication(ctx, o, oi.Clone(), sizeS)
 	}
 	return size
 }
@@ -1063,7 +1094,7 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj Ob
 	return action
 }
 
-func applyTransitionRule(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) bool {
+func applyTransitionRule(obj ObjectInfo) bool {
 	if obj.DeleteMarker {
 		return false
 	}
@@ -1089,7 +1120,9 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 }
 
 func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
-	opts := ObjectOptions{}
+	opts := ObjectOptions{
+		Expiration: ExpirationOptions{Expire: true},
+	}
 
 	if applyOnVersion {
 		opts.VersionID = obj.VersionID
@@ -1128,22 +1161,20 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 }
 
 // Apply object, object version, restored object or restored object version action on the given object
-func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
+func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
-	if obj.TransitionedObject.Status != "" {
+	globalExpiryState.enqueueByDays(obj, restoredObject, applyOnVersion)
-		return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject)
+	return true
-	}
-	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion)
 }
 
 // Perform actions (removal or transitioning of objects), return true the action is successfully performed
-func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) {
+func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-		success = applyExpiryRule(ctx, objLayer, obj, false, action == lifecycle.DeleteVersionAction)
+		success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
 	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		success = applyExpiryRule(ctx, objLayer, obj, true, action == lifecycle.DeleteRestoredVersionAction)
+		success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		success = applyTransitionRule(ctx, action, objLayer, obj)
+		success = applyTransitionRule(obj)
 	}
 	return
 }
@@ -1155,74 +1186,95 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
-	existingObjResync := i.replication.Resync(ctx, oi)
+	roi := getHealReplicateObjectInfo(oi, i.replication)
+
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		// heal delete marker replication failure or versioned delete replication failure
 		if oi.ReplicationStatus == replication.Pending ||
 			oi.ReplicationStatus == replication.Failed ||
 			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
+			i.healReplicationDeletes(ctx, o, roi)
 			return
 		}
 		// if replication status is Complete on DeleteMarker and existing object resync required
-		if existingObjResync && (oi.ReplicationStatus == replication.Completed) {
+		if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
-			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
+			i.healReplicationDeletes(ctx, o, roi)
 			return
 		}
 		return
 	}
-	roi := ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType}
+	if roi.ExistingObjResync.mustResync() {
-	if existingObjResync {
 		roi.OpType = replication.ExistingObjectReplicationType
-		roi.ResetID = i.replication.ResetID
 	}
-	switch oi.ReplicationStatus {
+
+	if roi.TargetStatuses != nil {
+		if sizeS.replTargetStats == nil {
+			sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
+		}
+		for arn, tgtStatus := range roi.TargetStatuses {
+			tgtSizeS, ok := sizeS.replTargetStats[arn]
+			if !ok {
+				tgtSizeS = replTargetSizeSummary{}
+			}
+			switch tgtStatus {
 			case replication.Pending:
+				tgtSizeS.pendingCount++
+				tgtSizeS.pendingSize += oi.Size
 				sizeS.pendingCount++
 				sizeS.pendingSize += oi.Size
-		globalReplicationPool.queueReplicaTask(roi)
-		return
 			case replication.Failed:
+				tgtSizeS.failedSize += oi.Size
+				tgtSizeS.failedCount++
 				sizeS.failedSize += oi.Size
 				sizeS.failedCount++
+			case replication.Completed, "COMPLETE":
+				tgtSizeS.replicatedSize += oi.Size
+				sizeS.replicatedSize += oi.Size
+			}
+			sizeS.replTargetStats[arn] = tgtSizeS
+		}
+	}
+
+	switch oi.ReplicationStatus {
+	case replication.Pending, replication.Failed:
 		globalReplicationPool.queueReplicaTask(roi)
 		return
-	case replication.Completed, "COMPLETE":
-		sizeS.replicatedSize += oi.Size
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
-	if existingObjResync {
+	if roi.ExistingObjResync.mustResync() {
 		globalReplicationPool.queueReplicaTask(roi)
 	}
 }
 
 // healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo, existingObject bool) {
+func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
 	// handle soft delete and permanent delete failures here.
-	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
 		versionID := ""
 		dmVersionID := ""
-		if oi.VersionPurgeStatus.Empty() {
+		if roi.VersionPurgeStatus.Empty() {
-			dmVersionID = oi.VersionID
+			dmVersionID = roi.VersionID
 		} else {
-			versionID = oi.VersionID
+			versionID = roi.VersionID
 		}
+
 		doi := DeletedObjectReplicationInfo{
 			DeletedObject: DeletedObject{
-				ObjectName:                    oi.Name,
+				ObjectName:            roi.Name,
 				DeleteMarkerVersionID: dmVersionID,
 				VersionID:             versionID,
-				DeleteMarkerReplicationStatus: string(oi.ReplicationStatus),
+				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-				DeleteMarkerMTime:             DeleteMarkerMTime{oi.ModTime},
+				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-				DeleteMarker:                  oi.DeleteMarker,
+				DeleteMarker:          roi.DeleteMarker,
-				VersionPurgeStatus:            oi.VersionPurgeStatus,
 			},
-			Bucket: oi.Bucket,
+			Bucket: roi.Bucket,
+			OpType: replication.HealReplicationType,
 		}
-		if existingObject {
+		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType
-			doi.ResetID = i.replication.ResetID
+			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
+			return
 		}
 		globalReplicationPool.queueReplicaDeleteTask(doi)
 	}

cmd/data-update-tracker.go

@@ -46,7 +46,7 @@ const (
 	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 6
+	dataUpdateTrackerVersion      = 7
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -203,7 +203,7 @@ func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
 // start a saver goroutine.
 // All of these will exit when the context is canceled.
 func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
-	if len(drives) <= 0 {
+	if len(drives) == 0 {
 		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No drives specified"))
 		return
 	}
@@ -220,7 +220,7 @@ func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
 // If no valid data usage tracker can be found d will remain unchanged.
 // If object is shared the caller should lock it.
 func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
-	if len(drives) <= 0 {
+	if len(drives) == 0 {
 		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No drives specified"))
 		return
 	}
@@ -397,7 +397,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2, 3, 4, 5:
+	case 1, 2, 3, 4, 5, 6:
 		if intDataUpdateTracker.debug {
 			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		}
@@ -473,7 +473,11 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for in := range d.input {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case in := <-d.input:
 			bucket, _ := path2BucketObjectWithBasePath("", in)
 			if bucket == "" {
 				if d.debug && len(in) > 0 {
@@ -496,6 +500,7 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 			d.mu.Unlock()
 		}
 	}
+}
 
 // markDirty adds the supplied path to the current bloom filter.
 func (d *dataUpdateTracker) markDirty(bucket, prefix string) {

cmd/data-usage-cache.go

@@ -46,20 +46,74 @@ type dataUsageHash string
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
 
-//msgp:tuple dataUsageEntry
 type dataUsageEntry struct {
-	Children dataUsageHashMap
+	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
-	Size             int64
+	Size             int64                `msg:"sz"`
-	Objects          uint64
+	Objects          uint64               `msg:"os"`
-	Versions         uint64 // Versions that are not delete markers.
+	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
-	ObjSizes         sizeHistogram
+	ObjSizes         sizeHistogram        `msg:"szs"`
-	ReplicationStats *replicationStats
+	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
-	Compacted        bool
+	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
+	Compacted        bool                 `msg:"c"`
 }
 
-//msgp:tuple replicationStats
+// allTierStats is a collection of per-tier stats across all configured remote
-type replicationStats struct {
+// tiers.
+type allTierStats struct {
+	Tiers map[string]tierStats `msg:"ts"`
+}
+
+func newAllTierStats() *allTierStats {
+	return &allTierStats{
+		Tiers: make(map[string]tierStats),
+	}
+}
+
+func (ats *allTierStats) addSizes(sz sizeSummary) {
+	for tier, st := range sz.tiers {
+		ats.Tiers[tier] = ats.Tiers[tier].add(st)
+	}
+}
+
+func (ats *allTierStats) merge(other *allTierStats) {
+	for tier, st := range other.Tiers {
+		ats.Tiers[tier] = ats.Tiers[tier].add(st)
+	}
+}
+
+func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {
+	if ats == nil {
+		return stats
+	}
+
+	// Update stats for tiers as they become available.
+	for tier, st := range ats.Tiers {
+		stats[tier] = madmin.TierStats{
+			TotalSize:   st.TotalSize,
+			NumVersions: st.NumVersions,
+			NumObjects:  st.NumObjects,
+		}
+	}
+	return stats
+}
+
+// tierStats holds per-tier stats of a remote tier.
+type tierStats struct {
+	TotalSize   uint64 `msg:"ts"`
+	NumVersions int    `msg:"nv"`
+	NumObjects  int    `msg:"no"`
+}
+
+func (ts tierStats) add(u tierStats) tierStats {
+	ts.TotalSize += u.TotalSize
+	ts.NumVersions += u.NumVersions
+	ts.NumObjects += u.NumObjects
+	return ts
+}
+
+//msgp:tuple replicationStatsV1
+type replicationStatsV1 struct {
 	PendingSize          uint64
 	ReplicatedSize       uint64
 	FailedSize           uint64
@@ -72,8 +126,44 @@ type replicationStats struct {
 	AfterThresholdCount  uint64
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4
+func (rsv1 replicationStatsV1) Empty() bool {
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4
+	return rsv1.ReplicatedSize == 0 &&
+		rsv1.FailedSize == 0 &&
+		rsv1.FailedCount == 0
+}
+
+//msgp:tuple replicationStats
+type replicationStats struct {
+	PendingSize          uint64
+	ReplicatedSize       uint64
+	FailedSize           uint64
+	FailedCount          uint64
+	PendingCount         uint64
+	MissedThresholdSize  uint64
+	AfterThresholdSize   uint64
+	MissedThresholdCount uint64
+	AfterThresholdCount  uint64
+}
+
+func (rs replicationStats) Empty() bool {
+	return rs.ReplicatedSize == 0 &&
+		rs.FailedSize == 0 &&
+		rs.FailedCount == 0
+}
+
+type replicationAllStats struct {
+	Targets     map[string]replicationStats `msg:"t,omitempty"`
+	ReplicaSize uint64                      `msg:"r,omitempty"`
+}
+
+//msgp:tuple replicationAllStatsV1
+type replicationAllStatsV1 struct {
+	Targets     map[string]replicationStats
+	ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`
+}
+
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -104,7 +194,31 @@ type dataUsageEntryV4 struct {
 	Size             int64
 	Objects          uint64
 	ObjSizes         sizeHistogram
-	ReplicationStats replicationStats
+	ReplicationStats replicationStatsV1
+}
+
+//msgp:tuple dataUsageEntryV5
+type dataUsageEntryV5 struct {
+	Children dataUsageHashMap
+	// These fields do no include any children.
+	Size             int64
+	Objects          uint64
+	Versions         uint64 // Versions that are not delete markers.
+	ObjSizes         sizeHistogram
+	ReplicationStats *replicationStatsV1
+	Compacted        bool
+}
+
+//msgp:tuple dataUsageEntryV6
+type dataUsageEntryV6 struct {
+	Children dataUsageHashMap
+	// These fields do no include any children.
+	Size             int64
+	Objects          uint64
+	Versions         uint64 // Versions that are not delete markers.
+	ObjSizes         sizeHistogram
+	ReplicationStats *replicationAllStatsV1
+	Compacted        bool
 }
 
 // dataUsageCache contains a cache of data usage entries latest version.
@@ -114,8 +228,8 @@ type dataUsageCache struct {
 	Disks []string
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -124,20 +238,34 @@ type dataUsageCacheV2 struct {
 	Cache map[string]dataUsageEntryV2
 }
 
-// dataUsageCache contains a cache of data usage entries version 3.
+// dataUsageCacheV3 contains a cache of data usage entries version 3.
 type dataUsageCacheV3 struct {
 	Info  dataUsageCacheInfo
 	Disks []string
 	Cache map[string]dataUsageEntryV3
 }
 
-// dataUsageCache contains a cache of data usage entries version 4.
+// dataUsageCacheV4 contains a cache of data usage entries version 4.
 type dataUsageCacheV4 struct {
 	Info  dataUsageCacheInfo
 	Disks []string
 	Cache map[string]dataUsageEntryV4
 }
 
+// dataUsageCacheV5 contains a cache of data usage entries version 5.
+type dataUsageCacheV5 struct {
+	Info  dataUsageCacheInfo
+	Disks []string
+	Cache map[string]dataUsageEntryV5
+}
+
+// dataUsageCacheV6 contains a cache of data usage entries version 6.
+type dataUsageCacheV6 struct {
+	Info  dataUsageCacheInfo
+	Disks []string
+	Cache map[string]dataUsageEntryV6
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -170,17 +298,34 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Versions += summary.versions
 	e.ObjSizes.add(summary.totalSize)
 
-	if summary.replicaSize > 0 || summary.pendingSize > 0 || summary.replicatedSize > 0 ||
-		summary.failedCount > 0 || summary.pendingCount > 0 || summary.failedSize > 0 {
 	if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationStats{}
+		e.ReplicationStats = &replicationAllStats{
+			Targets: make(map[string]replicationStats),
+		}
+	} else if e.ReplicationStats.Targets == nil {
+		e.ReplicationStats.Targets = make(map[string]replicationStats)
 	}
-		e.ReplicationStats.ReplicatedSize += uint64(summary.replicatedSize)
-		e.ReplicationStats.FailedSize += uint64(summary.failedSize)
-		e.ReplicationStats.PendingSize += uint64(summary.pendingSize)
 	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
-		e.ReplicationStats.PendingCount += summary.pendingCount
+
-		e.ReplicationStats.FailedCount += summary.failedCount
+	if summary.replTargetStats != nil {
+		for arn, st := range summary.replTargetStats {
+			tgtStat, ok := e.ReplicationStats.Targets[arn]
+			if !ok {
+				tgtStat = replicationStats{}
+			}
+			tgtStat.PendingSize += uint64(st.pendingSize)
+			tgtStat.FailedSize += uint64(st.failedSize)
+			tgtStat.ReplicatedSize += uint64(st.replicatedSize)
+			tgtStat.FailedCount += st.failedCount
+			tgtStat.PendingCount += st.pendingCount
+			e.ReplicationStats.Targets[arn] = tgtStat
+		}
+	}
+	if summary.tiers != nil {
+		if e.AllTierStats == nil {
+			e.AllTierStats = newAllTierStats()
+		}
+		e.AllTierStats.addSizes(summary)
 	}
 }
 
@@ -189,24 +334,35 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Versions += other.Versions
 	e.Size += other.Size
-	ors := other.ReplicationStats
+	if other.ReplicationStats != nil {
-	empty := replicationStats{}
-	if ors != nil && *ors != empty {
 		if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationStats{}
+			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
+		} else if e.ReplicationStats.Targets == nil {
+			e.ReplicationStats.Targets = make(map[string]replicationStats)
 		}
-		e.ReplicationStats.PendingSize += other.ReplicationStats.PendingSize
-		e.ReplicationStats.FailedSize += other.ReplicationStats.FailedSize
-		e.ReplicationStats.ReplicatedSize += other.ReplicationStats.ReplicatedSize
 		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
-		e.ReplicationStats.PendingCount += other.ReplicationStats.PendingCount
+		for arn, stat := range other.ReplicationStats.Targets {
-		e.ReplicationStats.FailedCount += other.ReplicationStats.FailedCount
+			st := e.ReplicationStats.Targets[arn]
-
+			e.ReplicationStats.Targets[arn] = replicationStats{
+				PendingSize:    stat.PendingSize + st.PendingSize,
+				FailedSize:     stat.FailedSize + st.FailedSize,
+				ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,
+				PendingCount:   stat.PendingCount + st.PendingCount,
+				FailedCount:    stat.FailedCount + st.FailedCount,
+			}
+		}
 	}
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
 	}
+
+	if other.AllTierStats != nil {
+		if e.AllTierStats == nil {
+			e.AllTierStats = newAllTierStats()
+		}
+		e.AllTierStats.merge(other.AllTierStats)
+	}
 }
 
 // mod returns true if the hash mod cycles == cycle.
@@ -253,6 +409,11 @@ func (e dataUsageEntry) clone() dataUsageEntry {
 		r := *e.ReplicationStats
 		e.ReplicationStats = &r
 	}
+	if e.AllTierStats != nil {
+		ats := newAllTierStats()
+		ats.merge(e.AllTierStats)
+		e.AllTierStats = ats
+	}
 	return e
 }
 
@@ -361,27 +522,20 @@ func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
 
 // dui converts the flattened version of the path to madmin.DataUsageInfo.
 // As a side effect d will be flattened, use a clone if this is not ok.
-func (d *dataUsageCache) dui(path string, buckets []BucketInfo) madmin.DataUsageInfo {
+func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	e := d.find(path)
 	if e == nil {
 		// No entry found, return empty.
-		return madmin.DataUsageInfo{}
+		return DataUsageInfo{}
 	}
 	flat := d.flatten(*e)
-	dui := madmin.DataUsageInfo{
+	dui := DataUsageInfo{
 		LastUpdate:        d.Info.LastUpdate,
 		ObjectsTotalCount: flat.Objects,
 		ObjectsTotalSize:  uint64(flat.Size),
 		BucketsCount:      uint64(len(e.Children)),
 		BucketsUsage:      d.bucketsUsageInfo(buckets),
-	}
+		TierStats:         d.tiersUsageInfo(buckets),
-	if flat.ReplicationStats != nil {
-		dui.ReplicationPendingSize = flat.ReplicationStats.PendingSize
-		dui.ReplicatedSize = flat.ReplicationStats.ReplicatedSize
-		dui.ReplicationFailedSize = flat.ReplicationStats.FailedSize
-		dui.ReplicationPendingCount = flat.ReplicationStats.PendingCount
-		dui.ReplicationFailedCount = flat.ReplicationStats.FailedCount
-		dui.ReplicaSize = flat.ReplicationStats.ReplicaSize
 	}
 	return dui
 }
@@ -598,28 +752,53 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
 	return res
 }
 
-// bucketsUsageInfo returns the buckets usage info as a map, with
+func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
-// key as bucket name
+	dst := newAllTierStats()
-func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]madmin.BucketUsageInfo {
-	var dst = make(map[string]madmin.BucketUsageInfo, len(buckets))
 	for _, bucket := range buckets {
 		e := d.find(bucket.Name)
 		if e == nil {
 			continue
 		}
 		flat := d.flatten(*e)
-		bui := madmin.BucketUsageInfo{
+		if flat.AllTierStats == nil {
+			continue
+		}
+		dst.merge(flat.AllTierStats)
+	}
+	if len(dst.Tiers) == 0 {
+		return nil
+	}
+	return dst
+}
+
+// bucketsUsageInfo returns the buckets usage info as a map, with
+// key as bucket name
+func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo {
+	var dst = make(map[string]BucketUsageInfo, len(buckets))
+	for _, bucket := range buckets {
+		e := d.find(bucket.Name)
+		if e == nil {
+			continue
+		}
+		flat := d.flatten(*e)
+		bui := BucketUsageInfo{
 			Size:                 uint64(flat.Size),
 			ObjectsCount:         flat.Objects,
 			ObjectSizesHistogram: flat.ObjSizes.toMap(),
 		}
 		if flat.ReplicationStats != nil {
-			bui.ReplicationPendingSize = flat.ReplicationStats.PendingSize
-			bui.ReplicatedSize = flat.ReplicationStats.ReplicatedSize
-			bui.ReplicationFailedSize = flat.ReplicationStats.FailedSize
-			bui.ReplicationPendingCount = flat.ReplicationStats.PendingCount
-			bui.ReplicationFailedCount = flat.ReplicationStats.FailedCount
 			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
+			bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
+			for arn, stat := range flat.ReplicationStats.Targets {
+				bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
+					ReplicationPendingSize:  stat.PendingSize,
+					ReplicatedSize:          stat.ReplicatedSize,
+					ReplicationFailedSize:   stat.FailedSize,
+					ReplicationPendingCount: stat.PendingCount,
+					ReplicationFailedCount:  stat.FailedCount,
+				}
+
+			}
 		}
 		dst[bucket.Name] = bui
 	}
@@ -628,24 +807,29 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]madmi
 
 // bucketUsageInfo returns the buckets usage info.
 // If not found all values returned are zero values.
-func (d *dataUsageCache) bucketUsageInfo(bucket string) madmin.BucketUsageInfo {
+func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
 	e := d.find(bucket)
 	if e == nil {
-		return madmin.BucketUsageInfo{}
+		return BucketUsageInfo{}
 	}
 	flat := d.flatten(*e)
-	bui := madmin.BucketUsageInfo{
+	bui := BucketUsageInfo{
 		Size:                 uint64(flat.Size),
 		ObjectsCount:         flat.Objects,
 		ObjectSizesHistogram: flat.ObjSizes.toMap(),
 	}
 	if flat.ReplicationStats != nil {
-		bui.ReplicationPendingSize = flat.ReplicationStats.PendingSize
-		bui.ReplicatedSize = flat.ReplicationStats.ReplicatedSize
-		bui.ReplicationFailedSize = flat.ReplicationStats.FailedSize
-		bui.ReplicationPendingCount = flat.ReplicationStats.PendingCount
-		bui.ReplicationFailedCount = flat.ReplicationStats.FailedCount
 		bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
+		bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
+		for arn, stat := range flat.ReplicationStats.Targets {
+			bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
+				ReplicationPendingSize:  stat.PendingSize,
+				ReplicatedSize:          stat.ReplicatedSize,
+				ReplicationFailedSize:   stat.FailedSize,
+				ReplicationPendingCount: stat.PendingCount,
+				ReplicationFailedCount:  stat.FailedCount,
+			}
+		}
 	}
 	return bui
 }
@@ -744,6 +928,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		case ObjectNotFound:
 		case BucketNotFound:
 		case InsufficientReadQuorum:
+		case StorageErr:
 		default:
 			return toObjectErr(err, dataUsageBucket, name)
 		}
@@ -789,7 +974,9 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 5
+	dataUsageCacheVerCurrent = 7
+	dataUsageCacheVerV6      = 6
+	dataUsageCacheVerV5      = 5
 	dataUsageCacheVerV4      = 4
 	dataUsageCacheVerV3      = 3
 	dataUsageCacheVerV2      = 2
@@ -884,13 +1071,19 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				Children: v.Children,
 			}
 			if v.ReplicatedSize > 0 || v.ReplicaSize > 0 || v.ReplicationFailedSize > 0 || v.ReplicationPendingSize > 0 {
-				due.ReplicationStats = &replicationStats{
+				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
+				if cfg != nil && cfg.RoleArn != "" {
+					due.ReplicationStats = &replicationAllStats{
+						Targets: make(map[string]replicationStats),
+					}
+					due.ReplicationStats.ReplicaSize = v.ReplicaSize
+					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
 						ReplicatedSize: v.ReplicatedSize,
-					ReplicaSize:    v.ReplicaSize,
 						FailedSize:     v.ReplicationFailedSize,
 						PendingSize:    v.ReplicationPendingSize,
 					}
 				}
+			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
 			d.Cache[k] = due
@@ -917,9 +1110,23 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			empty := replicationStats{}
+			empty := replicationStatsV1{}
+
 			if v.ReplicationStats != empty {
-				due.ReplicationStats = &v.ReplicationStats
+				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
+				if cfg != nil && cfg.RoleArn != "" {
+					due.ReplicationStats = &replicationAllStats{
+						Targets: make(map[string]replicationStats),
+					}
+					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
+						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
+						FailedSize:     v.ReplicationStats.FailedSize,
+						FailedCount:    v.ReplicationStats.FailedCount,
+						PendingSize:    v.ReplicationStats.PendingSize,
+						PendingCount:   v.ReplicationStats.PendingCount,
+					}
+					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
+				}
 			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
@@ -927,15 +1134,99 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		}
 
 		// Populate compacted value and remove unneeded replica stats.
-		empty := replicationStats{}
 		for k, e := range d.Cache {
-			if e.ReplicationStats != nil && *e.ReplicationStats == empty {
+			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
 				e.ReplicationStats = nil
 			}
-
 			d.Cache[k] = e
 		}
 		return nil
+	case dataUsageCacheVerV5:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		dold := &dataUsageCacheV5{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Disks = dold.Disks
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			due := dataUsageEntry{
+				Size:     v.Size,
+				Objects:  v.Objects,
+				ObjSizes: v.ObjSizes,
+				Children: v.Children,
+			}
+			if v.ReplicationStats != nil && !v.ReplicationStats.Empty() {
+				cfg, _ := getReplicationConfig(GlobalContext, d.Info.Name)
+				if cfg != nil && cfg.RoleArn != "" {
+					due.ReplicationStats = &replicationAllStats{
+						Targets: make(map[string]replicationStats),
+					}
+					d.Info.replication = replicationConfig{Config: cfg}
+
+					due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
+						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
+						FailedSize:     v.ReplicationStats.FailedSize,
+						FailedCount:    v.ReplicationStats.FailedCount,
+						PendingSize:    v.ReplicationStats.PendingSize,
+						PendingCount:   v.ReplicationStats.PendingCount,
+					}
+					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
+				}
+			}
+			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
+
+			d.Cache[k] = due
+		}
+
+		// Populate compacted value and remove unneeded replica stats.
+		for k, e := range d.Cache {
+			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
+				e.ReplicationStats = nil
+			}
+			d.Cache[k] = e
+		}
+		return nil
+	case dataUsageCacheVerV6:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		dold := &dataUsageCacheV6{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Disks = dold.Disks
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			var replicationStats *replicationAllStats
+			if v.ReplicationStats != nil {
+				replicationStats = &replicationAllStats{
+					Targets:     v.ReplicationStats.Targets,
+					ReplicaSize: v.ReplicationStats.ReplicaSize,
+				}
+			}
+			due := dataUsageEntry{
+				Children:         v.Children,
+				Size:             v.Size,
+				Objects:          v.Objects,
+				Versions:         v.Versions,
+				ObjSizes:         v.ObjSizes,
+				ReplicationStats: replicationStats,
+				Compacted:        v.Compacted,
+			}
+			d.Cache[k] = due
+		}
+		return nil
 	case dataUsageCacheVerCurrent:
 		// Zstd compressed.
 		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))

cmd/data-usage-cache_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalallTierStats(t *testing.T) {
+	v := allTierStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgallTierStats(b *testing.B) {
+	v := allTierStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgallTierStats(b *testing.B) {
+	v := allTierStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalallTierStats(b *testing.B) {
+	v := allTierStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeallTierStats(t *testing.T) {
+	v := allTierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeallTierStats Msgsize() is inaccurate")
+	}
+
+	vn := allTierStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeallTierStats(b *testing.B) {
+	v := allTierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeallTierStats(b *testing.B) {
+	v := allTierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageCache(t *testing.T) {
 	v := dataUsageCache{}
 	bts, err := v.MarshalMsg(nil)
@@ -348,6 +461,232 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalreplicationAllStats(t *testing.T) {
+	v := replicationAllStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgreplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgreplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalreplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodereplicationAllStats(t *testing.T) {
+	v := replicationAllStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodereplicationAllStats Msgsize() is inaccurate")
+	}
+
+	vn := replicationAllStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodereplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodereplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalreplicationAllStatsV1(t *testing.T) {
+	v := replicationAllStatsV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgreplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgreplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalreplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodereplicationAllStatsV1(t *testing.T) {
+	v := replicationAllStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodereplicationAllStatsV1 Msgsize() is inaccurate")
+	}
+
+	vn := replicationAllStatsV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodereplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodereplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalreplicationStats(t *testing.T) {
 	v := replicationStats{}
 	bts, err := v.MarshalMsg(nil)
@@ -461,6 +800,119 @@ func BenchmarkDecodereplicationStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalreplicationStatsV1(t *testing.T) {
+	v := replicationStatsV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgreplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgreplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalreplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodereplicationStatsV1(t *testing.T) {
+	v := replicationStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodereplicationStatsV1 Msgsize() is inaccurate")
+	}
+
+	vn := replicationStatsV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodereplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodereplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
 	v := sizeHistogram{}
 	bts, err := v.MarshalMsg(nil)
@@ -573,3 +1025,116 @@ func BenchmarkDecodesizeHistogram(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshaltierStats(t *testing.T) {
+	v := tierStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgtierStats(b *testing.B) {
+	v := tierStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgtierStats(b *testing.B) {
+	v := tierStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaltierStats(b *testing.B) {
+	v := tierStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodetierStats(t *testing.T) {
+	v := tierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodetierStats Msgsize() is inaccurate")
+	}
+
+	vn := tierStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodetierStats(b *testing.B) {
+	v := tierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodetierStats(b *testing.B) {
+	v := tierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage-utils.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"sort"
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+// BucketTargetUsageInfo - bucket target usage info provides
+// - replicated size for all objects sent to this target
+// - replica size for all objects received from this target
+// - replication pending size for all objects pending replication to this target
+// - replication failed size for all objects failed replication to this target
+// - replica pending count
+// - replica failed count
+type BucketTargetUsageInfo struct {
+	ReplicationPendingSize  uint64 `json:"objectsPendingReplicationTotalSize"`
+	ReplicationFailedSize   uint64 `json:"objectsFailedReplicationTotalSize"`
+	ReplicatedSize          uint64 `json:"objectsReplicatedTotalSize"`
+	ReplicaSize             uint64 `json:"objectReplicaTotalSize"`
+	ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"`
+	ReplicationFailedCount  uint64 `json:"objectsFailedReplicationCount"`
+}
+
+// BucketUsageInfo - bucket usage info provides
+// - total size of the bucket
+// - total objects in a bucket
+// - object size histogram per bucket
+type BucketUsageInfo struct {
+	Size uint64 `json:"size"`
+	// Following five fields suffixed with V1 are here for backward compatibility
+	// Total Size for objects that have not yet been replicated
+	ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
+	// Total size for objects that have witness one or more failures and will be retried
+	ReplicationFailedSizeV1 uint64 `json:"objectsFailedReplicationTotalSize"`
+	// Total size for objects that have been replicated to destination
+	ReplicatedSizeV1 uint64 `json:"objectsReplicatedTotalSize"`
+	// Total number of objects pending replication
+	ReplicationPendingCountV1 uint64 `json:"objectsPendingReplicationCount"`
+	// Total number of objects that failed replication
+	ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
+
+	ObjectsCount         uint64                           `json:"objectsCount"`
+	ObjectSizesHistogram map[string]uint64                `json:"objectsSizesHistogram"`
+	ReplicaSize          uint64                           `json:"objectReplicaTotalSize"`
+	ReplicationInfo      map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+}
+
+// DataUsageInfo represents data usage stats of the underlying Object API
+type DataUsageInfo struct {
+	// LastUpdate is the timestamp of when the data usage info was last updated.
+	// This does not indicate a full scan.
+	LastUpdate time.Time `json:"lastUpdate"`
+
+	// Objects total count across all buckets
+	ObjectsTotalCount uint64 `json:"objectsCount"`
+
+	// Objects total size across all buckets
+	ObjectsTotalSize uint64                           `json:"objectsTotalSize"`
+	ReplicationInfo  map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+	// Total number of buckets in this cluster
+	BucketsCount uint64 `json:"bucketsCount"`
+
+	// Buckets usage info provides following information across all buckets
+	// - total size of the bucket
+	// - total objects in a bucket
+	// - object size histogram per bucket
+	BucketsUsage map[string]BucketUsageInfo `json:"bucketsUsageInfo"`
+	// Deprecated kept here for backward compatibility reasons.
+	BucketSizes map[string]uint64 `json:"bucketsSizes"`
+
+	// TierStats contains per-tier stats of all configured remote tiers
+	TierStats *allTierStats `json:"tierStats,omitempty"`
+}
+
+func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
+	if globalTierConfigMgr.Empty() {
+		return nil
+	}
+
+	ts := make(map[string]madmin.TierStats)
+	// Add configured remote tiers
+	for tier := range globalTierConfigMgr.Tiers {
+		ts[tier] = madmin.TierStats{}
+	}
+	// Add STANDARD (hot-tier)
+	ts[minioHotTier] = madmin.TierStats{}
+
+	ts = dui.TierStats.adminStats(ts)
+	infos := make([]madmin.TierInfo, 0, len(ts))
+	for tier, st := range ts {
+		var tierType string
+		if tier == minioHotTier {
+			tierType = "internal"
+		} else {
+			tierType = globalTierConfigMgr.Tiers[tier].Type.String()
+		}
+		infos = append(infos, madmin.TierInfo{
+			Name:  tier,
+			Type:  tierType,
+			Stats: st,
+		})
+	}
+
+	sort.Slice(infos, func(i, j int) bool {
+		if infos[i].Type == "internal" {
+			return true
+		}
+		if infos[j].Type == "internal" {
+			return false
+		}
+		return infos[i].Name < infos[j].Name
+	})
+	return infos
+}

cmd/data-usage.go

@@ -24,7 +24,6 @@ import (
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 )
@@ -39,7 +38,7 @@ const (
 )
 
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan madmin.DataUsageInfo) {
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
 	for dataUsageInfo := range dui {
 		var json = jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
@@ -95,27 +94,26 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 	return m, nil
 }
 
-func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.DataUsageInfo, error) {
+func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
-			return madmin.DataUsageInfo{}, nil
+			return DataUsageInfo{}, nil
 		}
-		return madmin.DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
+		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
 	defer r.Close()
 
-	var dataUsageInfo madmin.DataUsageInfo
+	var dataUsageInfo DataUsageInfo
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.NewDecoder(r).Decode(&dataUsageInfo); err != nil {
-		return madmin.DataUsageInfo{}, err
+		return DataUsageInfo{}, err
 	}
-
 	// For forward compatibility reasons, we need to add this code.
 	if len(dataUsageInfo.BucketsUsage) == 0 {
-		dataUsageInfo.BucketsUsage = make(map[string]madmin.BucketUsageInfo, len(dataUsageInfo.BucketSizes))
+		dataUsageInfo.BucketsUsage = make(map[string]BucketUsageInfo, len(dataUsageInfo.BucketSizes))
 		for bucket, size := range dataUsageInfo.BucketSizes {
-			dataUsageInfo.BucketsUsage[bucket] = madmin.BucketUsageInfo{Size: size}
+			dataUsageInfo.BucketsUsage[bucket] = BucketUsageInfo{Size: size}
 		}
 	}
 
@@ -126,6 +124,22 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.D
 			dataUsageInfo.BucketSizes[bucket] = bui.Size
 		}
 	}
-
+	// For forward compatibility reasons, we need to add this code.
+	for bucket, bui := range dataUsageInfo.BucketsUsage {
+		if bui.ReplicatedSizeV1 > 0 || bui.ReplicationFailedCountV1 > 0 ||
+			bui.ReplicationFailedSizeV1 > 0 || bui.ReplicationPendingCountV1 > 0 {
+			cfg, _ := getReplicationConfig(GlobalContext, bucket)
+			if cfg != nil && cfg.RoleArn != "" {
+				dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo)
+				dataUsageInfo.ReplicationInfo[cfg.RoleArn] = BucketTargetUsageInfo{
+					ReplicationFailedSize:   bui.ReplicationFailedSizeV1,
+					ReplicationFailedCount:  bui.ReplicationFailedCountV1,
+					ReplicatedSize:          bui.ReplicatedSizeV1,
+					ReplicationPendingCount: bui.ReplicationPendingCountV1,
+					ReplicationPendingSize:  bui.ReplicationPendingSizeV1,
+				}
+			}
+		}
+	}
 	return dataUsageInfo, nil
 }

cmd/data-usage_test.go

@@ -179,6 +179,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
 		got, err = scanDataFolder(context.Background(), base, got, getSize)
+		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -423,6 +424,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
 		got, err = scanDataFolder(context.Background(), base, got, getSize)
+		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -578,17 +580,16 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		t.Fatal(err)
 	}
 	e := want.find("abucket/dir2")
-	e.ReplicationStats = &replicationStats{
+	e.ReplicationStats = &replicationAllStats{
+		Targets: map[string]replicationStats{
+			"arn": {
 				PendingSize:    1,
 				ReplicatedSize: 2,
 				FailedSize:     3,
-		ReplicaSize:          4,
 				FailedCount:    5,
 				PendingCount:   6,
-		MissedThresholdSize:  7,
+			},
-		AfterThresholdSize:   8,
+		},
-		MissedThresholdCount: 9,
-		AfterThresholdCount:  10,
 	}
 	want.replace("abucket/dir2", "", *e)
 	var buf bytes.Buffer

cmd/disk-cache-utils.go

@@ -110,7 +110,7 @@ func cacheControlOpts(o ObjectInfo) *cacheControl {
 
 	var headerVal string
 	for k, v := range m {
-		if strings.ToLower(k) == "cache-control" {
+		if strings.EqualFold(k, "cache-control") {
 			headerVal = v
 		}
 
@@ -246,6 +246,9 @@ func decryptCacheObjectETag(info *ObjectInfo) error {
 		if globalCacheKMS == nil {
 			return errKMSNotConfigured
 		}
+		if len(info.Parts) > 0 { // multipart ETag is not encrypted since it is not md5sum
+			return nil
+		}
 		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(info.UserDefined)
 		if err != nil {
 			return err
@@ -271,6 +274,43 @@ func decryptCacheObjectETag(info *ObjectInfo) error {
 	return nil
 }
 
+// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
+func decryptCachePartETags(c *cacheMeta) ([]string, error) {
+	var partETags []string
+	encrypted := crypto.S3.IsEncrypted(c.Meta) && isCacheEncrypted(c.Meta)
+
+	switch {
+	case encrypted:
+		if globalCacheKMS == nil {
+			return partETags, errKMSNotConfigured
+		}
+		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(c.Meta)
+		if err != nil {
+			return partETags, err
+		}
+		extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{c.Bucket: path.Join(c.Bucket, c.Object)})
+		if err != nil {
+			return partETags, err
+		}
+		var objectKey crypto.ObjectKey
+		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), c.Bucket, c.Object); err != nil {
+			return partETags, err
+		}
+		for i := range c.PartETags {
+			etagStr := tryDecryptETag(objectKey[:], c.PartETags[i], false)
+			// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
+			etag, err := hex.DecodeString(etagStr)
+			if err != nil {
+				return []string{}, err
+			}
+			partETags = append(partETags, string(etag))
+		}
+		return partETags, nil
+	default:
+		return c.PartETags, nil
+	}
+}
+
 func isMetadataSame(m1, m2 map[string]string) bool {
 	if m1 == nil && m2 == nil {
 		return true
@@ -379,6 +419,9 @@ func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
 // Returns true if there still is a need to delete files (n+saveBytes >0),
 // false if no more bytes needs to be saved.
 func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	if f == nil {
+		return false
+	}
 	if int64(f.saveBytes)+n <= 0 {
 		f.saveBytes = 0
 		f.trimQueue()
@@ -506,3 +549,49 @@ func bytesToClear(total, free int64, quotaPct, lowWatermark, highWatermark uint6
 	lowWMUsage := total * (int64)(lowWatermark*quotaPct) / (100 * 100)
 	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
 }
+
+type multiWriter struct {
+	backendWriter io.Writer
+	cacheWriter   *io.PipeWriter
+	pipeClosed    bool
+}
+
+// multiWriter writes to backend and cache - if cache write
+// fails close the pipe, but continue writing to the backend
+func (t *multiWriter) Write(p []byte) (n int, err error) {
+	n, err = t.backendWriter.Write(p)
+	if err == nil && n != len(p) {
+		err = io.ErrShortWrite
+		return
+	}
+	if err != nil {
+		if !t.pipeClosed {
+			t.cacheWriter.CloseWithError(err)
+		}
+		return
+	}
+
+	// ignore errors writing to cache
+	if !t.pipeClosed {
+		_, cerr := t.cacheWriter.Write(p)
+		if cerr != nil {
+			t.pipeClosed = true
+			t.cacheWriter.CloseWithError(cerr)
+		}
+	}
+	return len(p), nil
+}
+func cacheMultiWriter(w1 io.Writer, w2 *io.PipeWriter) io.Writer {
+	return &multiWriter{backendWriter: w1, cacheWriter: w2}
+}
+
+// writebackInProgress returns true if writeback commit is not complete
+func writebackInProgress(m map[string]string) bool {
+	if v, ok := m[writeBackStatusHeader]; ok {
+		switch cacheCommitStatus(v) {
+		case CommitPending, CommitFailed:
+			return true
+		}
+	}
+	return false
+}

cmd/disk-cache.go

@@ -59,6 +59,13 @@ const (
 	CommitFailed cacheCommitStatus = "failed"
 )
 
+const (
+	// CommitWriteBack allows staging and write back of cached content for single object uploads
+	CommitWriteBack string = "writeback"
+	// CommitWriteThrough allows caching multipart uploads to disk synchronously
+	CommitWriteThrough string = "writethrough"
+)
+
 // String returns string representation of status
 func (s cacheCommitStatus) String() string {
 	return string(s)
@@ -80,6 +87,13 @@ type CacheObjectLayer interface {
 	DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
+	// Multipart operations.
+	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
+	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error
+	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error)
+
 	// Storage operations.
 	StorageInfo(ctx context.Context) CacheStorageInfo
 	CacheStats() *CacheStats
@@ -95,10 +109,10 @@ type cacheObjects struct {
 	after int
 	// commit objects in async manner
 	commitWriteback    bool
+	commitWritethrough bool
+
 	// if true migration is in progress from v1 to v2
 	migrating bool
-	// mutex to protect migration bool
-	migMutex sync.Mutex
 	// retry queue for writeback cache mode to reattempt upload to backend
 	wbRetryCh chan ObjectInfo
 	// Cache stats
@@ -109,11 +123,16 @@ type cacheObjects struct {
 	InnerDeleteObjectFn            func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	InnerPutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	InnerCopyObjectFn              func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerNewMultipartUploadFn      func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
+	InnerPutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
+	InnerAbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error
+	InnerCompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerCopyObjectPartFn          func(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error)
 }
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
 	metadata := map[string]string{"etag": eTag}
-	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true)
+	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true, false)
 }
 
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
@@ -140,7 +159,7 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 		bkObjectInfo.ETag != cacheObjInfo.ETag ||
 		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
 		!bkObjectInfo.Expires.Equal(cacheObjInfo.Expires) {
-		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false)
+		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false, false)
 	}
 	return c.incHitsToMeta(ctx, dcache, bucket, object, cacheObjInfo.Size, cacheObjInfo.ETag, rs)
 }
@@ -257,6 +276,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			bReader.ObjInfo.CacheStatus = CacheMiss
 			return bReader, err
 		}
+		// serve cached content without ETag verification if writeback commit is not yet complete
+		if writebackInProgress(cacheReader.ObjInfo.UserDefined) {
+			return cacheReader, nil
+		}
 	}
 
 	objInfo, err := c.InnerGetObjectInfoFn(ctx, bucket, object, opts)
@@ -349,7 +372,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 				// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
 				dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{
 					UserDefined: getMetadata(bReader.ObjInfo),
-				}, false)
+				}, false, false)
 				return
 			}
 		}()
@@ -367,7 +390,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			io.LimitReader(pr, bkReader.ObjInfo.Size),
 			bkReader.ObjInfo.Size, rs, ObjectOptions{
 				UserDefined: userDefined,
-			}, false)
+			}, false, false)
 		// close the read end of the pipe, so the error gets
 		// propagated to teeReader
 		pr.CloseWithError(putErr)
@@ -403,6 +426,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 			c.cacheStats.incHit()
 			return cachedObjInfo, nil
 		}
+		// serve cache metadata without ETag verification if writeback commit is not yet complete
+		if writebackInProgress(cachedObjInfo.UserDefined) {
+			return cachedObjInfo, nil
+		}
 	}
 
 	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
@@ -488,8 +515,6 @@ func (c *cacheObjects) CacheStats() (cs *CacheStats) {
 
 // skipCache() returns true if cache migration is in progress
 func (c *cacheObjects) skipCache() bool {
-	c.migMutex.Lock()
-	defer c.migMutex.Unlock()
 	return c.migrating
 }
 
@@ -619,8 +644,6 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	}
 
 	// update migration status
-	c.migMutex.Lock()
-	defer c.migMutex.Unlock()
 	c.migrating = false
 	logStartupMessage(color.Blue("Cache migration completed successfully."))
 }
@@ -663,15 +686,15 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	if c.commitWriteback {
-		oi, err := dcache.Put(ctx, bucket, object, r, r.Size(), nil, opts, false)
+		oi, err := dcache.Put(ctx, bucket, object, r, r.Size(), nil, opts, false, true)
 		if err != nil {
 			return ObjectInfo{}, err
 		}
 		go c.uploadObject(GlobalContext, oi)
 		return oi, nil
 	}
+	if !c.commitWritethrough {
 		objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
-
 		if err == nil {
 			go func() {
 				// fill cache in the background
@@ -683,12 +706,63 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 				oi, _, err := dcache.Stat(GlobalContext, bucket, object)
 				// avoid cache overwrite if another background routine filled cache
 				if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, false)
 				}
 			}()
 		}
 		return objInfo, err
 	}
+	cLock, lkctx, cerr := dcache.GetLockContext(GlobalContext, bucket, object)
+	if cerr != nil {
+		return putObjectFn(ctx, bucket, object, r, opts)
+	}
+	defer cLock.Unlock(lkctx.Cancel)
+	// Initialize pipe to stream data to backend
+	pipeReader, pipeWriter := io.Pipe()
+	hashReader, err := hash.NewReader(pipeReader, size, "", "", r.ActualSize())
+	if err != nil {
+		return
+	}
+	// Initialize pipe to stream data to cache
+	rPipe, wPipe := io.Pipe()
+	infoCh := make(chan ObjectInfo)
+	errorCh := make(chan error)
+	go func() {
+		info, err := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader), opts)
+		if err != nil {
+			close(infoCh)
+			pipeReader.CloseWithError(err)
+			rPipe.CloseWithError(err)
+			errorCh <- err
+			return
+		}
+		close(errorCh)
+		infoCh <- info
+	}()
+
+	go func() {
+		_, err := dcache.put(lkctx.Context(), bucket, object, rPipe, r.Size(), nil, opts, false, false)
+		if err != nil {
+			rPipe.CloseWithError(err)
+			return
+		}
+	}()
+
+	mwriter := cacheMultiWriter(pipeWriter, wPipe)
+	_, err = io.Copy(mwriter, r)
+	pipeWriter.Close()
+	wPipe.Close()
+
+	if err != nil {
+		err = <-errorCh
+		return ObjectInfo{}, err
+	}
+	info := <-infoCh
+	if cerr = dcache.updateMetadata(lkctx.Context(), bucket, object, info.ETag, info.ModTime, info.Size); cerr != nil {
+		dcache.delete(bucket, object)
+	}
+	return info, err
+}
 
 // upload cached object to backend in async commit mode.
 func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
@@ -720,6 +794,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
 	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader), opts)
 	wbCommitStatus := CommitComplete
+	size := objInfo.Size
 	if err != nil {
 		wbCommitStatus = CommitFailed
 	}
@@ -730,12 +805,13 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 		retryCnt, _ = strconv.Atoi(meta[writeBackRetryHeader])
 		retryCnt++
 		meta[writeBackRetryHeader] = strconv.Itoa(retryCnt)
+		size = cReader.ObjInfo.Size
 	} else {
 		delete(meta, writeBackRetryHeader)
 	}
 	meta[writeBackStatusHeader] = wbCommitStatus.String()
 	meta["etag"] = oi.ETag
-	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, objInfo.Size, nil, "", false)
+	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, size, nil, "", false, wbCommitStatus == CommitComplete)
 	if retryCnt > 0 {
 		// slow down retries
 		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
@@ -745,6 +821,8 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 
 func (c *cacheObjects) queueWritebackRetry(oi ObjectInfo) {
 	select {
+	case <-GlobalContext.Done():
+		return
 	case c.wbRetryCh <- oi:
 		c.uploadObject(GlobalContext, oi)
 	default:
@@ -763,8 +841,9 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		exclude:            config.Exclude,
 		after:              config.After,
 		migrating:          migrateSw,
-		migMutex:        sync.Mutex{},
+		commitWriteback:    config.CacheCommitMode == CommitWriteBack,
-		commitWriteback: config.CommitWriteback,
+		commitWritethrough: config.CacheCommitMode == CommitWriteThrough,
+
 		cacheStats: newCacheStats(),
 		InnerGetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
@@ -781,6 +860,21 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		InnerCopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
+		InnerNewMultipartUploadFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
+			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, opts)
+		},
+		InnerPutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
+			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
+		},
+		InnerAbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+			return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
+		},
+		InnerCompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
+		},
+		InnerCopyObjectPartFn: func(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
+			return newObjectLayerFn().CopyObjectPart(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length, srcInfo, srcOpts, dstOpts)
+		},
 	}
 	c.cacheStats.GetDiskStats = func() []CacheDiskStats {
 		cacheDiskStats := make([]CacheDiskStats, len(c.cache))
@@ -859,3 +953,247 @@ func (c *cacheObjects) queuePendingWriteback(ctx context.Context) {
 		}
 	}
 }
+
+// NewMultipartUpload - Starts a new multipart upload operation to backend - if writethrough mode is enabled, starts caching the multipart.
+func (c *cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
+	newMultipartUploadFn := c.InnerNewMultipartUploadFn
+	dcache, err := c.getCacheToLoc(ctx, bucket, object)
+	if err != nil {
+		// disk cache could not be located,execute backend call.
+		return newMultipartUploadFn(ctx, bucket, object, opts)
+	}
+	if c.skipCache() {
+		return newMultipartUploadFn(ctx, bucket, object, opts)
+	}
+
+	if opts.ServerSideEncryption != nil { // avoid caching encrypted objects
+		dcache.Delete(ctx, bucket, object)
+		return newMultipartUploadFn(ctx, bucket, object, opts)
+	}
+
+	// skip cache for objects with locks
+	objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
+	if objRetention.Mode.Valid() || legalHold.Status.Valid() {
+		dcache.Delete(ctx, bucket, object)
+		return newMultipartUploadFn(ctx, bucket, object, opts)
+	}
+
+	// fetch from backend if cache exclude pattern or cache-control
+	// directive set to exclude
+	if c.isCacheExclude(bucket, object) {
+		dcache.Delete(ctx, bucket, object)
+		return newMultipartUploadFn(ctx, bucket, object, opts)
+	}
+	if !c.commitWritethrough && !c.commitWriteback {
+		return newMultipartUploadFn(ctx, bucket, object, opts)
+	}
+
+	// perform multipart upload on backend and cache simultaneously
+	uploadID, err = newMultipartUploadFn(ctx, bucket, object, opts)
+	dcache.NewMultipartUpload(GlobalContext, bucket, object, uploadID, opts)
+	return uploadID, err
+}
+
+// PutObjectPart streams part to cache concurrently if writethrough mode is enabled. Otherwise redirects the call to remote
+func (c *cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
+	putObjectPartFn := c.InnerPutObjectPartFn
+	dcache, err := c.getCacheToLoc(ctx, bucket, object)
+	if err != nil {
+		// disk cache could not be located,execute backend call.
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+
+	if !c.commitWritethrough && !c.commitWriteback {
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+	if c.skipCache() {
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+	size := data.Size()
+
+	// avoid caching part if space unavailable
+	if !dcache.diskSpaceAvailable(size) {
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+
+	if opts.ServerSideEncryption != nil {
+		dcache.Delete(ctx, bucket, object)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+
+	// skip cache for objects with locks
+	objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
+	if objRetention.Mode.Valid() || legalHold.Status.Valid() {
+		dcache.Delete(ctx, bucket, object)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+
+	// fetch from backend if cache exclude pattern or cache-control
+	// directive set to exclude
+	if c.isCacheExclude(bucket, object) {
+		dcache.Delete(ctx, bucket, object)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
+	}
+
+	info = PartInfo{}
+	// Initialize pipe to stream data to backend
+	pipeReader, pipeWriter := io.Pipe()
+	hashReader, err := hash.NewReader(pipeReader, size, "", "", data.ActualSize())
+	if err != nil {
+		return
+	}
+	// Initialize pipe to stream data to cache
+	rPipe, wPipe := io.Pipe()
+	pinfoCh := make(chan PartInfo)
+	cinfoCh := make(chan PartInfo)
+
+	errorCh := make(chan error)
+	go func() {
+		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, NewPutObjReader(hashReader), opts)
+		if err != nil {
+			close(pinfoCh)
+			pipeReader.CloseWithError(err)
+			rPipe.CloseWithError(err)
+			errorCh <- err
+			return
+		}
+		close(errorCh)
+		pinfoCh <- info
+	}()
+	go func() {
+		pinfo, perr := dcache.PutObjectPart(GlobalContext, bucket, object, uploadID, partID, rPipe, data.Size(), opts)
+		if perr != nil {
+			rPipe.CloseWithError(perr)
+			close(cinfoCh)
+			// clean up upload
+			dcache.AbortUpload(bucket, object, uploadID)
+			return
+		}
+		cinfoCh <- pinfo
+	}()
+
+	mwriter := cacheMultiWriter(pipeWriter, wPipe)
+	_, err = io.Copy(mwriter, data)
+	pipeWriter.Close()
+	wPipe.Close()
+
+	if err != nil {
+		err = <-errorCh
+		return PartInfo{}, err
+	}
+	info = <-pinfoCh
+	cachedInfo := <-cinfoCh
+	if info.PartNumber == cachedInfo.PartNumber {
+		cachedInfo.ETag = info.ETag
+		cachedInfo.LastModified = info.LastModified
+		dcache.SavePartMetadata(GlobalContext, bucket, object, uploadID, partID, cachedInfo)
+	}
+	return info, err
+}
+
+// CopyObjectPart behaves similar to PutObjectPart - caches part to upload dir if writethrough mode is enabled.
+func (c *cacheObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
+	copyObjectPartFn := c.InnerCopyObjectPartFn
+	dcache, err := c.getCacheToLoc(ctx, dstBucket, dstObject)
+	if err != nil {
+		// disk cache could not be located,execute backend call.
+		return copyObjectPartFn(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length, srcInfo, srcOpts, dstOpts)
+	}
+
+	if !c.commitWritethrough && !c.commitWriteback {
+		return copyObjectPartFn(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length, srcInfo, srcOpts, dstOpts)
+	}
+	if err := dcache.uploadIDExists(dstBucket, dstObject, uploadID); err != nil {
+		return copyObjectPartFn(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length, srcInfo, srcOpts, dstOpts)
+	}
+	partInfo, err := copyObjectPartFn(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, startOffset, length, srcInfo, srcOpts, dstOpts)
+	if err != nil {
+		return pi, toObjectErr(err, dstBucket, dstObject)
+	}
+	go func() {
+		isSuffixLength := false
+		if startOffset < 0 {
+			isSuffixLength = true
+		}
+
+		rs := &HTTPRangeSpec{
+			IsSuffixLength: isSuffixLength,
+			Start:          startOffset,
+			End:            startOffset + length,
+		}
+		// fill cache in the background
+		bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, srcBucket, srcObject, rs, http.Header{}, readLock, ObjectOptions{})
+		if bErr != nil {
+			return
+		}
+		defer bReader.Close()
+		// avoid cache overwrite if another background routine filled cache
+		dcache.PutObjectPart(GlobalContext, dstBucket, dstObject, uploadID, partID, bReader, length, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+	}()
+	// Success.
+	return partInfo, nil
+}
+
+// CompleteMultipartUpload - completes multipart upload operation on the backend. If writethrough mode is enabled, this also
+// finalizes the upload saved in cache multipart dir.
+func (c *cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
+	completeMultipartUploadFn := c.InnerCompleteMultipartUploadFn
+	if !c.commitWritethrough && !c.commitWriteback {
+		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
+	}
+	dcache, err := c.getCacheToLoc(ctx, bucket, object)
+	if err != nil {
+		// disk cache could not be located,execute backend call.
+		return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
+	}
+
+	// perform multipart upload on backend and cache simultaneously
+	oi, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
+	if err == nil {
+		// fill cache in the background
+		go func() {
+			_, err := dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, oi, opts)
+			if err != nil {
+				// fill cache in the background
+				bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+				if bErr != nil {
+					return
+				}
+				defer bReader.Close()
+				oi, _, err := dcache.Stat(GlobalContext, bucket, object)
+				// avoid cache overwrite if another background routine filled cache
+				if err != nil || oi.ETag != bReader.ObjInfo.ETag {
+					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, false)
+				}
+			}
+		}()
+	}
+	return
+}
+
+// AbortMultipartUpload - aborts multipart upload on backend and cache.
+func (c *cacheObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+	abortMultipartUploadFn := c.InnerAbortMultipartUploadFn
+	if !c.commitWritethrough && !c.commitWriteback {
+		return abortMultipartUploadFn(ctx, bucket, object, uploadID, opts)
+	}
+	dcache, err := c.getCacheToLoc(ctx, bucket, object)
+	if err != nil {
+		// disk cache could not be located,execute backend call.
+		return abortMultipartUploadFn(ctx, bucket, object, uploadID, opts)
+	}
+	if err = dcache.uploadIDExists(bucket, object, uploadID); err != nil {
+		return toObjectErr(err, bucket, object, uploadID)
+	}
+
+	// execute backend operation
+	err = abortMultipartUploadFn(ctx, bucket, object, uploadID, opts)
+	if err != nil {
+		return err
+	}
+	// abort multipart upload on cache
+	go dcache.AbortUpload(bucket, object, uploadID)
+	return nil
+}

cmd/disk-cache_test.go

@@ -25,7 +25,7 @@ import (
 // Tests ToObjectInfo function.
 func TestCacheMetadataObjInfo(t *testing.T) {
 	m := cacheMeta{Meta: nil}
-	objInfo := m.ToObjectInfo("testbucket", "testobject")
+	objInfo := m.ToObjectInfo()
 	if objInfo.Size != 0 {
 		t.Fatal("Unexpected object info value for Size", objInfo.Size)
 	}

cmd/dummy-data-generator_test.go

@@ -61,7 +61,7 @@ func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker {
 		panic("Negative rotations are not allowed")
 	}
 
-	skipOffset = skipOffset % int64(len(alphabets))
+	skipOffset %= int64(len(alphabets))
 	as := make([]byte, 2*len(alphabets))
 	copy(as, alphabets)
 	copy(as[len(alphabets):], alphabets)

cmd/dummy-handlers.go

@@ -164,7 +164,6 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 // DeleteBucketWebsiteHandler - DELETE bucket website, a dummy api
 func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseHeadersOnly(w)
-	w.(http.Flusher).Flush()
 }
 
 // GetBucketCorsHandler - GET bucket cors, a dummy api

cmd/encryption-v1.go

@@ -68,13 +68,30 @@ const (
 
 )
 
-// isEncryptedMultipart returns true if the current object is
+// KMSKeyID returns in AWS compatible KMS KeyID() format.
+func (o ObjectInfo) KMSKeyID() string {
+	if len(o.UserDefined) == 0 {
+		return ""
+	}
+	kmsID, ok := o.UserDefined[crypto.MetaKeyID]
+	if !ok {
+		return ""
+	}
+	if strings.HasPrefix(kmsID, "arn:aws:kms:") {
+		return kmsID
+	}
+	return "arn:aws:kms:" + kmsID
+}
+
+// isMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
-func (o *ObjectInfo) isEncryptedMultipart() bool {
+func (o *ObjectInfo) isMultipart() bool {
 	if len(o.Parts) == 0 {
 		return false
 	}
+	_, encrypted := crypto.IsEncrypted(o.UserDefined)
+	if encrypted {
 		if !crypto.IsMultiPart(o.UserDefined) {
 			return false
 		}
@@ -84,6 +101,8 @@ func (o *ObjectInfo) isEncryptedMultipart() bool {
 				return false
 			}
 		}
+	}
+
 	// Further check if this object is uploaded using multipart mechanism
 	// by the user and it is not about Erasure internally splitting the
 	// object into parts in PutObject()
@@ -427,7 +446,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint3
 
 	bucket, object := oi.Bucket, oi.Name
 	// Single part case
-	if !oi.isEncryptedMultipart() {
+	if !oi.isMultipart() {
 		var reader io.Reader
 		var err error
 		if copySource {
@@ -589,7 +608,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	if !o.isEncryptedMultipart() {
+	if !o.isMultipart() {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
 			err = errObjectTampered // assign correct error type
@@ -732,7 +751,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	// Assemble slice of (decrypted) part sizes in `sizes`
 	var sizes []int64
 	var decObjSize int64 // decrypted total object size
-	if o.isEncryptedMultipart() {
+	if o.isMultipart() {
 		sizes = make([]int64, len(o.Parts))
 		for i, part := range o.Parts {
 			var partSize uint64

cmd/endpoint-ellipses.go

@@ -242,7 +242,7 @@ func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 {
 	for _, argPattern := range argPatterns {
 		var totalSize uint64 = 1
 		for _, p := range argPattern {
-			totalSize = totalSize * uint64(len(p.Seq))
+			totalSize *= uint64(len(p.Seq))
 		}
 		totalSizes = append(totalSizes, totalSize)
 	}

cmd/endpoint.go

@@ -18,8 +18,6 @@
 package cmd
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -36,7 +34,6 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mountinfo"
 	"github.com/minio/pkg/env"
@@ -500,6 +497,7 @@ func NewEndpoints(args ...string) (endpoints Endpoints, err error) {
 		}
 
 		// All endpoints have to be same type and scheme if applicable.
+		//nolint:gocritic
 		if i == 0 {
 			endpointType = endpoint.Type()
 			scheme = endpoint.Scheme
@@ -743,72 +741,6 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	return -1
 }
 
-func httpDo(clnt *http.Client, req *http.Request, f func(*http.Response, error) error) error {
-	ctx, cancel := context.WithTimeout(GlobalContext, 200*time.Millisecond)
-	defer cancel()
-
-	// Run the HTTP request in a goroutine and pass the response to f.
-	c := make(chan error, 1)
-	req = req.WithContext(ctx)
-	go func() { c <- f(clnt.Do(req)) }()
-	select {
-	case <-ctx.Done():
-		<-c // Wait for f to return.
-		return ctx.Err()
-	case err := <-c:
-		return err
-	}
-}
-
-func getOnlineProxyEndpointIdx() int {
-	type reqIndex struct {
-		Request *http.Request
-		Idx     int
-	}
-
-	proxyRequests := make(map[*http.Client]reqIndex, len(globalProxyEndpoints))
-	for i, proxyEp := range globalProxyEndpoints {
-		proxyEp := proxyEp
-		serverURL := &url.URL{
-			Scheme: proxyEp.Scheme,
-			Host:   proxyEp.Host,
-			Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
-		}
-
-		req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
-		if err != nil {
-			continue
-		}
-
-		proxyRequests[&http.Client{
-			Transport: proxyEp.Transport,
-		}] = reqIndex{
-			Request: req,
-			Idx:     i,
-		}
-	}
-
-	for c, r := range proxyRequests {
-		if err := httpDo(c, r.Request, func(resp *http.Response, err error) error {
-			if err != nil {
-				return err
-			}
-			xhttp.DrainBody(resp.Body)
-			if resp.StatusCode != http.StatusOK {
-				return errors.New(resp.Status)
-			}
-			if v := resp.Header.Get(xhttp.MinIOServerStatus); v == unavailable {
-				return errors.New(v)
-			}
-			return nil
-		}); err != nil {
-			continue
-		}
-		return r.Idx
-	}
-	return -1
-}
-
 // GetProxyEndpoints - get all endpoints that can be used to proxy list request.
 func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint {
 	var proxyEps []ProxyEndpoint