fix: inherit heal opts globally, including bitrot settings (#11166)

Bonus re-use ReadFileStream internal io.Copy buffers, fixes
lots of chatty allocations when reading metacache readers
with many sustained concurrent listing operations

```
   17.30GB  1.27% 84.80%    35.26GB  2.58%  io.copyBuffer
```

.github/workflows/codeql.yml

@@ -1,51 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-  pull_request:
-  schedule:
-    - cron: '0 19 * * 0'
-
-jobs:
-  CodeQL-Build:
-
-    # CodeQL runs on ubuntu-latest and windows-latest
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-      
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-         languages: go, javascript
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/go.yml

@@ -26,7 +26,6 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_CI_CD: 1
         run: |
           go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
           go test -v --timeout 50m ./...
@@ -35,7 +34,6 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_CI_CD: 1
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0

.golangci.yml

@@ -23,5 +23,11 @@ issues:
   exclude:
       - should have a package comment
       - error strings should not be capitalized or end with punctuation or a newline
+
+run:
+  skip-dirs:
+    - pkg/rpc
+    - pkg/argon2
+
 service:
   golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine as builder
+FROM golang:1.15-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
@@ -11,22 +11,25 @@ RUN  \
      git clone https://github.com/minio/minio && cd minio && \
      git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 EXPOSE 9000
 
 COPY --from=builder /go/bin/minio /usr/bin/minio
-COPY --from=builder /go/minio/CREDITS /third_party/
+COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
+COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
 COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     microdnf clean all && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

Dockerfile.arm.release

@@ -1,34 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-arm as qemu
-FROM arm32v7/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
-
-ENV MINIO_UPDATE off
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.arm64.release

@@ -1,35 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-aarch64 as qemu
-FROM arm64v8/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
-
-ENV MINIO_UPDATE off
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
-     /usr/bin/verify-minio.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.cicd

@@ -0,0 +1,40 @@
+FROM golang:1.15-alpine as builder
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+    apk add --no-cache git && \
+    git clone https://github.com/minio/minio && cd minio && \
+    git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
+
+ARG TARGETARCH
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
+
+EXPOSE 9000
+
+COPY --from=builder /go/bin/minio /usr/bin/minio
+COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
+COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
+COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
+
+RUN  \
+    microdnf update --nodocs && \
+    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+    microdnf clean all
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio", "server", "/data"]

Dockerfile.dev

@@ -1,22 +1,26 @@
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
+
+ARG TARGETARCH
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
-COPY CREDITS /third_party/
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
 
-ENV MINIO_UPDATE off
+ENV MINIO_UPDATE=off \
-ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
-RUN \
+RUN  \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+    microdnf update --nodocs && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
-     chmod +x /usr/bin/minio  && \
+    microdnf clean all && \
-     chmod +x /usr/bin/docker-entrypoint.sh
+    chmod +x /usr/bin/minio  && \
+    chmod +x /usr/bin/docker-entrypoint.sh
 
 EXPOSE 9000
 

Dockerfile.dev.browser

@@ -1,4 +1,4 @@
-FROM ubuntu
+FROM ubuntu:20.04
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Dockerfile.ppc64le.release

@@ -1,34 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-ppc64le as qemu
-FROM ppc64le/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin
-
-ENV MINIO_UPDATE off
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
-     /usr/bin/verify-minio.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Dockerfile.release

@@ -1,26 +1,38 @@
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
-LABEL maintainer="MinIO Inc <dev@min.io>"
+ARG TARGETARCH
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="RELEASE.2020-11-25T22-36-25Z" \
+      release="RELEASE.2020-11-25T22-36-25Z" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" 
+
+COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
 
 RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+     microdnf update --nodocs && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/bin/minio && \
+     microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     microdnf clean all && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

Dockerfile.s390x.release

@@ -1,34 +0,0 @@
-FROM multiarch/qemu-user-static:x86_64-s390x as qemu
-FROM s390x/alpine:3.12
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin
-
-ENV MINIO_UPDATE off
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
-
-RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio.minisig -o /usr/bin/minio.minisig && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
-     /usr/bin/verify-minio.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Makefile

@@ -57,7 +57,7 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
@@ -71,6 +71,10 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
+hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
+hotfix: install
+
 docker: checks
 	@echo "Building minio docker image '$(TAG)'"
 	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

README.md

@@ -88,7 +88,7 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.14](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio
@@ -150,7 +150,7 @@ service iptables restart
 ```
 
 ## Test using MinIO Browser
-MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
+MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure your server has started successfully.
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 

VULNERABILITY_REPORT.md

@@ -0,0 +1,39 @@
+## Vulnerability Management Policy
+
+This document formally describes the process of addressing and managing a
+reported vulnerability that has been found in the MinIO server code base,
+any directly connected ecosystem component or a direct / indirect dependency
+of the code base.
+
+### Scope
+
+The vulnerability management policy described in this document covers the
+process of investigating, assessing and resolving a vulnerability report
+opened by a MinIO employee or an external third party.
+
+Therefore, it lists pre-conditions and actions that should be performed to
+resolve and fix a reported vulnerability.
+
+### Vulnerability Management Process
+
+The vulnerability management process requires that the vulnerability report
+contains the following information:
+
+ - The project / component that contains the reported vulnerability.
+ - A description of the vulnerability. In particular, the type of the
+   reported vulnerability and how it might be exploited. Alternatively,
+   a well-established vulnerability identifier, e.g. CVE number, can be
+   used instead.
+
+Based on the description mentioned above, a MinIO engineer or security team
+member investigates:
+
+ - Whether the reported vulnerability exists.
+ - The conditions that are required such that the vulnerability can be exploited.
+ - The steps required to fix the vulnerability.
+
+In general, if the vulnerability exists in one of the MinIO code bases
+itself - not in a code dependency - then MinIO will, if possible, fix
+the vulnerability or implement reasonable countermeasures such that the
+vulnerability cannot be exploited anymore.
+

browser/app/js/web.js

@@ -21,7 +21,7 @@ import storage from 'local-storage-fallback'
 
 class Web {
   constructor(endpoint) {
-    const namespace = 'Web'
+    const namespace = 'web'
     this.JSONrpc = new JSONrpc({
       endpoint,
       namespace

browser/package-lock.json

@@ -2271,7 +2271,9 @@
     "asap": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
+      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=",
+      "dev": true,
+      "optional": true
     },
     "asn1": {
       "version": "0.2.4",
@@ -2283,19 +2285,20 @@
       }
     },
     "asn1.js": {
-      "version": "4.10.1",
+      "version": "5.4.1",
-      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",
+      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
-      "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+      "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
       "requires": {
         "bn.js": "^4.0.0",
         "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0"
+        "minimalistic-assert": "^1.0.0",
+        "safer-buffer": "^2.1.0"
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
+          "version": "4.11.9",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -3541,9 +3544,9 @@
       }
     },
     "base64-js": {
-      "version": "1.3.1",
+      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
     },
     "batch": {
       "version": "0.6.1",
@@ -3574,13 +3577,12 @@
     "bluebird": {
       "version": "3.7.2",
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
-      "dev": true
     },
     "bn.js": {
-      "version": "5.1.1",
+      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz",
-      "integrity": "sha512-IUTD/REb78Z2eodka1QZyyEk66pciRcP6Sroka0aI3tG/iwIdYLrBD62RsubR7vqdt3WyX8p4jxeatzmRSphtA=="
+      "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ=="
     },
     "body-parser": {
       "version": "1.19.0",
@@ -3754,31 +3756,24 @@
       }
     },
     "browserify-rsa": {
-      "version": "4.0.1",
+      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz",
-      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
+      "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==",
       "requires": {
-        "bn.js": "^4.1.0",
+        "bn.js": "^5.0.0",
         "randombytes": "^2.0.1"
-      },
-      "dependencies": {
-        "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
-        }
       }
     },
     "browserify-sign": {
-      "version": "4.2.0",
+      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz",
-      "integrity": "sha512-hEZC1KEeYuoHRqhGhTy6gWrpJA3ZDjFWv0DE61643ZnOXAKJb3u7yWcrU0mMc9SwAqK1n7myPGndkp0dFG7NFA==",
+      "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==",
       "requires": {
         "bn.js": "^5.1.1",
         "browserify-rsa": "^4.0.1",
         "create-hash": "^1.2.0",
         "create-hmac": "^1.1.7",
-        "elliptic": "^6.5.2",
+        "elliptic": "^6.5.3",
         "inherits": "^2.0.4",
         "parse-asn1": "^5.1.5",
         "readable-stream": "^3.6.0",
@@ -4022,9 +4017,9 @@
       }
     },
     "chokidar": {
-      "version": "3.4.0",
+      "version": "3.4.3",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.3.tgz",
-      "integrity": "sha512-aXAaho2VJtisB/1fg1+3nlLJqGOuewTzQpd/Tz0yTg2R0e4IGtshYvtjowyEumcBv2z+y4+kc75Mz7j5xJskcQ==",
+      "integrity": "sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ==",
       "optional": true,
       "requires": {
         "anymatch": "~3.1.1",
@@ -4034,7 +4029,7 @@
         "is-binary-path": "~2.1.0",
         "is-glob": "~4.0.1",
         "normalize-path": "~3.0.0",
-        "readdirp": "~3.4.0"
+        "readdirp": "~3.5.0"
       },
       "dependencies": {
         "anymatch": {
@@ -4048,9 +4043,9 @@
           }
         },
         "binary-extensions": {
-          "version": "2.0.0",
+          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
-          "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
+          "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
           "optional": true
         },
         "braces": {
@@ -4123,9 +4118,9 @@
           "optional": true
         },
         "readdirp": {
-          "version": "3.4.0",
+          "version": "3.5.0",
-          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
-          "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+          "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
           "optional": true,
           "requires": {
             "picomatch": "^2.2.1"
@@ -4143,9 +4138,9 @@
       }
     },
     "chownr": {
-      "version": "1.1.1",
+      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
     },
     "chrome-trace-event": {
       "version": "1.0.2",
@@ -4713,18 +4708,18 @@
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
     "create-ecdh": {
-      "version": "4.0.3",
+      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
-      "integrity": "sha512-GbEHQPMOswGpKXM9kCWVrremUcBmjteUaQ01T9rkKCPDXfUHX0IoP9LpHYo2NPFampa4e+/pFDc3jQdxrxQLaw==",
+      "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==",
       "requires": {
         "bn.js": "^4.1.0",
-        "elliptic": "^6.0.0"
+        "elliptic": "^6.5.3"
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
+          "version": "4.11.9",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -4925,9 +4920,9 @@
       }
     },
     "cyclist": {
-      "version": "0.2.2",
+      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-0.2.2.tgz",
+      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
-      "integrity": "sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA="
+      "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk="
     },
     "dashdash": {
       "version": "1.14.1",
@@ -5242,9 +5237,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
+          "version": "4.11.9",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -5420,9 +5415,9 @@
       }
     },
     "duplexify": {
-      "version": "3.6.1",
+      "version": "3.7.1",
-      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
-      "integrity": "sha512-vM58DwdnKmty+FSPzT14K9JXb90H+j5emaR4KYbr2KTIz00WHGbWOe5ghQTx233ZCLZtrGDALzKwcjEtSt35mA==",
+      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
       "requires": {
         "end-of-stream": "^1.0.0",
         "inherits": "^2.0.1",
@@ -5471,9 +5466,9 @@
       "dev": true
     },
     "elliptic": {
-      "version": "6.5.2",
+      "version": "6.5.3",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
-      "integrity": "sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw==",
+      "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
       "requires": {
         "bn.js": "^4.4.0",
         "brorand": "^1.0.1",
@@ -5485,9 +5480,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
+          "version": "4.11.9",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -5509,14 +5504,6 @@
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=",
       "dev": true
     },
-    "encoding": {
-      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
-      "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
-      "requires": {
-        "iconv-lite": "~0.4.13"
-      }
-    },
     "end-of-stream": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
@@ -5526,9 +5513,9 @@
       }
     },
     "enhanced-resolve": {
-      "version": "4.1.1",
+      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz",
-      "integrity": "sha512-98p2zE+rL7/g/DzMHMTF4zZlCgeVdJ7yr6xzEpJRYwFYrGi9ANdn5DnJURg6RpBkyk60XYDnWIv51VfIhfNGuA==",
+      "integrity": "sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==",
       "requires": {
         "graceful-fs": "^4.1.2",
         "memory-fs": "^0.5.0",
@@ -5886,9 +5873,9 @@
       },
       "dependencies": {
         "lodash": {
-          "version": "4.17.15",
+          "version": "4.17.20",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         },
         "react-is": {
@@ -6162,11 +6149,18 @@
       "dev": true
     },
     "esrecurse": {
-      "version": "4.2.1",
+      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
-      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
       "requires": {
-        "estraverse": "^4.1.0"
+        "estraverse": "^5.2.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ=="
+        }
       }
     },
     "estraverse": {
@@ -6193,9 +6187,9 @@
       "dev": true
     },
     "events": {
-      "version": "3.1.0",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz",
-      "integrity": "sha512-Rv+u8MLHNOdMjTAFeT3nCjHn2aGlx435FP/sDHNaRhDEMwyI/aB22Kj2qIN8R0cw3z28psEQLYwxVKLsKrMgWg=="
+      "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg=="
     },
     "eventsource": {
       "version": "1.0.7",
@@ -6722,27 +6716,6 @@
         "bser": "2.1.1"
       }
     },
-    "fbjs": {
-      "version": "0.8.16",
-      "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.16.tgz",
-      "integrity": "sha1-XmdDL1UNxBtXK/VYR7ispk5TN9s=",
-      "requires": {
-        "core-js": "^1.0.0",
-        "isomorphic-fetch": "^2.1.1",
-        "loose-envify": "^1.0.0",
-        "object-assign": "^4.1.0",
-        "promise": "^7.1.1",
-        "setimmediate": "^1.0.5",
-        "ua-parser-js": "^0.7.9"
-      },
-      "dependencies": {
-        "core-js": {
-          "version": "1.2.7",
-          "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz",
-          "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY="
-        }
-      }
-    },
     "figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -7013,12 +6986,12 @@
       }
     },
     "flush-write-stream": {
-      "version": "1.0.3",
+      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
-      "integrity": "sha512-calZMC10u0FMUqoiunI2AiGIIUtUIvifNwkHhNupZH4cbNnW1Itkoh/Nf5HFYmDrwWPjrUxpkZT0KhuCq0jmGw==",
+      "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==",
       "requires": {
-        "inherits": "^2.0.1",
+        "inherits": "^2.0.3",
-        "readable-stream": "^2.0.4"
+        "readable-stream": "^2.3.6"
       }
     },
     "follow-redirects": {
@@ -8376,9 +8349,9 @@
           }
         },
         "lodash": {
-          "version": "4.17.15",
+          "version": "4.17.20",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         },
         "tapable": {
@@ -8651,14 +8624,6 @@
       "resolved": "https://registry.npmjs.org/humanize/-/humanize-0.0.9.tgz",
       "integrity": "sha1-GZT/rs3+nEQe0r2sdFK3u0yeQaQ="
     },
-    "iconv-lite": {
-      "version": "0.4.23",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
-      "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==",
-      "requires": {
-        "safer-buffer": ">= 2.1.2 < 3"
-      }
-    },
     "icss-utils": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-4.1.1.tgz",
@@ -8677,9 +8642,9 @@
       }
     },
     "ieee754": {
-      "version": "1.1.13",
+      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-      "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
     },
     "iferr": {
       "version": "0.1.5",
@@ -8746,9 +8711,9 @@
       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
     },
     "ini": {
-      "version": "1.3.5",
+      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
       "dev": true
     },
     "internal-ip": {
@@ -9013,7 +8978,8 @@
     "is-stream": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
+      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
+      "dev": true
     },
     "is-subset": {
       "version": "0.1.1",
@@ -9068,15 +9034,6 @@
       "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
       "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
     },
-    "isomorphic-fetch": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz",
-      "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=",
-      "requires": {
-        "node-fetch": "^1.0.1",
-        "whatwg-fetch": ">=0.10.0"
-      }
-    },
     "isstream": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
@@ -10573,9 +10530,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.14",
+      "version": "4.17.20",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
-      "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
       "dev": true
     },
     "lodash._baseisequal": {
@@ -10682,6 +10639,14 @@
         "tslib": "^1.10.0"
       }
     },
+    "lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "requires": {
+        "yallist": "^3.0.2"
+      }
+    },
     "make-dir": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
@@ -10868,9 +10833,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
+          "version": "4.11.9",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -10993,6 +10958,23 @@
         }
       }
     },
+    "mississippi": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
+      "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
+      "requires": {
+        "concat-stream": "^1.5.0",
+        "duplexify": "^3.4.2",
+        "end-of-stream": "^1.1.0",
+        "flush-write-stream": "^1.0.0",
+        "from2": "^2.1.0",
+        "parallel-transform": "^1.1.0",
+        "pump": "^3.0.0",
+        "pumpify": "^1.3.3",
+        "stream-each": "^1.1.0",
+        "through2": "^2.0.0"
+      }
+    },
     "mixin-deep": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
@@ -11165,19 +11147,10 @@
         "tslib": "^1.10.0"
       }
     },
-    "node-fetch": {
-      "version": "1.6.3",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.6.3.tgz",
-      "integrity": "sha1-3CNO3WSJmC1Y6PDbT2lQKavNjAQ=",
-      "requires": {
-        "encoding": "^0.1.11",
-        "is-stream": "^1.0.1"
-      }
-    },
     "node-forge": {
-      "version": "0.9.0",
+      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.0.tgz",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
-      "integrity": "sha512-7ASaDa3pD+lJ3WvXFsxekJQelBKRpne+GOVbLbtHYdd7pFspyeuJHnWfLplGf3SwKGbfs/aYl5V/JCIaHVUKKQ==",
+      "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==",
       "dev": true
     },
     "node-int64": {
@@ -11768,11 +11741,11 @@
       "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
     },
     "parallel-transform": {
-      "version": "1.1.0",
+      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz",
-      "integrity": "sha1-1BDwZbBdojCB/NEPKIVMKb2jOwY=",
+      "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==",
       "requires": {
-        "cyclist": "~0.2.2",
+        "cyclist": "^1.0.1",
         "inherits": "^2.0.3",
         "readable-stream": "^2.1.5"
       }
@@ -11788,13 +11761,12 @@
       }
     },
     "parse-asn1": {
-      "version": "5.1.5",
+      "version": "5.1.6",
-      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.5.tgz",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
-      "integrity": "sha512-jkMYn1dcJqF6d5CpU689bq7w/b5ALS9ROVSpQDPrZsqqesUJii9qutvoT5ltGedNXMO2e16YUWIghG9KxaViTQ==",
+      "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==",
       "requires": {
-        "asn1.js": "^4.0.0",
+        "asn1.js": "^5.2.0",
         "browserify-aes": "^1.0.0",
-        "create-hash": "^1.1.0",
         "evp_bytestokey": "^1.0.0",
         "pbkdf2": "^3.0.3",
         "safe-buffer": "^5.1.1"
@@ -11913,9 +11885,9 @@
       }
     },
     "pbkdf2": {
-      "version": "3.0.17",
+      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.17.tgz",
+      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
-      "integrity": "sha512-U/il5MsrZp7mGg3mSQfn742na2T+1/vHDCG5/iTI3X9MKUuYUZVLQhyRsg06mCgDBTd57TxzgZt7P+fYfjRLtA==",
+      "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==",
       "requires": {
         "create-hash": "^1.1.2",
         "create-hmac": "^1.1.4",
@@ -12218,6 +12190,8 @@
       "version": "7.3.1",
       "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
       "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
+      "dev": true,
+      "optional": true,
       "requires": {
         "asap": "~2.0.3"
       }
@@ -12238,13 +12212,23 @@
       }
     },
     "prop-types": {
-      "version": "15.6.0",
+      "version": "15.7.2",
-      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.0.tgz",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
-      "integrity": "sha1-zq8IMCL8RrSjX2nhPvda7Q1jmFY=",
+      "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
       "requires": {
-        "fbjs": "^0.8.16",
+        "loose-envify": "^1.4.0",
-        "loose-envify": "^1.3.1",
+        "object-assign": "^4.1.1",
-        "object-assign": "^4.1.1"
+        "react-is": "^16.8.1"
+      },
+      "dependencies": {
+        "loose-envify": {
+          "version": "1.4.0",
+          "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
+          "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
+          "requires": {
+            "js-tokens": "^3.0.0 || ^4.0.0"
+          }
+        }
       }
     },
     "prop-types-exact": {
@@ -12330,16 +12314,16 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
+          "version": "4.11.9",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
     "pump": {
-      "version": "2.0.1",
+      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
       "requires": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -12353,6 +12337,17 @@
         "duplexify": "^3.6.0",
         "inherits": "^2.0.3",
         "pump": "^2.0.0"
+      },
+      "dependencies": {
+        "pump": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
+          "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        }
       }
     },
     "punycode": {
@@ -13492,9 +13487,9 @@
       },
       "dependencies": {
         "lodash": {
-          "version": "4.17.15",
+          "version": "4.17.20",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         }
       }
@@ -13958,12 +13953,12 @@
       "dev": true
     },
     "selfsigned": {
-      "version": "1.10.7",
+      "version": "1.10.8",
-      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.7.tgz",
+      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.8.tgz",
-      "integrity": "sha512-8M3wBCzeWIJnQfl43IKwOmC4H/RAp50S8DF60znzjW5GVqTcSe2vWclt7hmYVPkKPlHWOu5EaWOMZ2Y6W8ZXTA==",
+      "integrity": "sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w==",
       "dev": true,
       "requires": {
-        "node-forge": "0.9.0"
+        "node-forge": "^0.10.0"
       }
     },
     "semver": {
@@ -14001,10 +13996,13 @@
       }
     },
     "serialize-javascript": {
-      "version": "3.0.0",
+      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.1.0.tgz",
-      "integrity": "sha512-skZcHYw2vEX4bw90nAr2iTTsz6x2SrHEnfxgKYmZlvJYBEZrvbKtobJWlQ20zczKb3bsHHXXTYt48zBA7ni9cw==",
+      "integrity": "sha512-JIJT1DGiWmIKhzRsG91aS6Ze4sFUrYbltlkg2onR5OrnNM02Kl/hnY/T4FN2omvyeBbQmMJv+K4cPOpGzOTFBg==",
-      "dev": true
+      "dev": true,
+      "requires": {
+        "randombytes": "^2.1.0"
+      }
     },
     "serializerr": {
       "version": "1.0.3",
@@ -14644,9 +14642,9 @@
       }
     },
     "stream-shift": {
-      "version": "1.0.0",
+      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
-      "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
+      "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ=="
     },
     "strict-uri-encode": {
       "version": "2.0.0",
@@ -15356,26 +15354,21 @@
       }
     },
     "terser-webpack-plugin": {
-      "version": "1.4.3",
+      "version": "1.4.5",
-      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.3.tgz",
+      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz",
-      "integrity": "sha512-QMxecFz/gHQwteWwSo5nTc6UaICqN1bMedC5sMtUc7y3Ha3Q8y6ZO0iCR8pq4RJC8Hjf0FEPEHZqcMB/+DFCrA==",
+      "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==",
       "requires": {
         "cacache": "^12.0.2",
         "find-cache-dir": "^2.1.0",
         "is-wsl": "^1.1.0",
         "schema-utils": "^1.0.0",
-        "serialize-javascript": "^2.1.2",
+        "serialize-javascript": "^4.0.0",
         "source-map": "^0.6.1",
         "terser": "^4.1.2",
         "webpack-sources": "^1.4.0",
         "worker-farm": "^1.7.0"
       },
       "dependencies": {
-        "bluebird": {
-          "version": "3.7.2",
-          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-          "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
-        },
         "cacache": {
           "version": "12.0.4",
           "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz",
@@ -15443,14 +15436,6 @@
             "path-exists": "^3.0.0"
           }
         },
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-          "requires": {
-            "yallist": "^3.0.2"
-          }
-        },
         "make-dir": {
           "version": "2.1.0",
           "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
@@ -15460,23 +15445,6 @@
             "semver": "^5.6.0"
           }
         },
-        "mississippi": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
-          "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
-          "requires": {
-            "concat-stream": "^1.5.0",
-            "duplexify": "^3.4.2",
-            "end-of-stream": "^1.1.0",
-            "flush-write-stream": "^1.0.0",
-            "from2": "^2.1.0",
-            "parallel-transform": "^1.1.0",
-            "pump": "^3.0.0",
-            "pumpify": "^1.3.3",
-            "stream-each": "^1.1.0",
-            "through2": "^2.0.0"
-          }
-        },
         "p-limit": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
@@ -15511,24 +15479,13 @@
             "find-up": "^3.0.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        },
         "serialize-javascript": {
-          "version": "2.1.2",
+          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-2.1.2.tgz",
+          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz",
-          "integrity": "sha512-rs9OggEUF0V4jUSecXazOYsLfu7OGK2qIn3c7IPBiffz32XniEp/TX9Xmc9LQfK2nQ2QKHvZ2oygKUGU0lG4jQ=="
+          "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==",
-        },
+          "requires": {
-        "source-list-map": {
+            "randombytes": "^2.1.0"
-          "version": "2.0.1",
+          }
-          "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
         },
         "source-map": {
           "version": "0.6.1",
@@ -15542,15 +15499,6 @@
           "requires": {
             "figgy-pudding": "^3.5.1"
           }
-        },
-        "webpack-sources": {
-          "version": "1.4.3",
-          "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
-          "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-          "requires": {
-            "source-list-map": "^2.0.0",
-            "source-map": "~0.6.1"
-          }
         }
       }
     },
@@ -15589,9 +15537,9 @@
       "dev": true
     },
     "timers-browserify": {
-      "version": "2.0.11",
+      "version": "2.0.12",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz",
-      "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==",
+      "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==",
       "requires": {
         "setimmediate": "^1.0.4"
       }
@@ -15790,11 +15738,6 @@
       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
       "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
     },
-    "ua-parser-js": {
-      "version": "0.7.20",
-      "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz",
-      "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw=="
-    },
     "uglify-js": {
       "version": "3.9.3",
       "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.9.3.tgz",
@@ -16172,20 +16115,20 @@
       }
     },
     "watchpack": {
-      "version": "1.7.2",
+      "version": "1.7.5",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.2.tgz",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz",
-      "integrity": "sha512-ymVbbQP40MFTp+cNMvpyBpBtygHnPzPkHqoIwRRj/0B8KhqQwV8LaKjtbaxF2lK4vl8zN9wCxS46IFCU5K4W0g==",
+      "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==",
       "requires": {
-        "chokidar": "^3.4.0",
+        "chokidar": "^3.4.1",
         "graceful-fs": "^4.1.2",
         "neo-async": "^2.5.0",
-        "watchpack-chokidar2": "^2.0.0"
+        "watchpack-chokidar2": "^2.0.1"
       }
     },
     "watchpack-chokidar2": {
-      "version": "2.0.0",
+      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz",
-      "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==",
+      "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==",
       "optional": true,
       "requires": {
         "chokidar": "^2.1.8"
@@ -16447,9 +16390,9 @@
       "dev": true
     },
     "webpack": {
-      "version": "4.43.0",
+      "version": "4.44.2",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.43.0.tgz",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.44.2.tgz",
-      "integrity": "sha512-GW1LjnPipFW2Y78OOab8NJlCflB7EFskMih2AHdvjbpKMeDJqEgSx24cXXXiPS65+WSwVyxtDsJH6jGX2czy+g==",
+      "integrity": "sha512-6KJVGlCxYdISyurpQ0IPTklv+DULv05rs2hseIXer6D7KrUicRDLFb4IUM1S6LUAKypPM/nSiVSuv8jHu1m3/Q==",
       "requires": {
         "@webassemblyjs/ast": "1.9.0",
         "@webassemblyjs/helper-module-context": "1.9.0",
@@ -16459,7 +16402,7 @@
         "ajv": "^6.10.2",
         "ajv-keywords": "^3.4.1",
         "chrome-trace-event": "^1.0.2",
-        "enhanced-resolve": "^4.1.0",
+        "enhanced-resolve": "^4.3.0",
         "eslint-scope": "^4.0.3",
         "json-parse-better-errors": "^1.0.2",
         "loader-runner": "^2.4.0",
@@ -16472,14 +16415,14 @@
         "schema-utils": "^1.0.0",
         "tapable": "^1.1.3",
         "terser-webpack-plugin": "^1.4.3",
-        "watchpack": "^1.6.1",
+        "watchpack": "^1.7.4",
         "webpack-sources": "^1.4.1"
       },
       "dependencies": {
         "ajv": {
-          "version": "6.12.2",
+          "version": "6.12.6",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-          "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==",
+          "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -16578,9 +16521,9 @@
           }
         },
         "fast-deep-equal": {
-          "version": "3.1.1",
+          "version": "3.1.3",
-          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
+          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
-          "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA=="
+          "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
         },
         "is-accessor-descriptor": {
           "version": "0.1.6",
@@ -16661,29 +16604,10 @@
             "to-regex": "^3.0.2"
           }
         },
-        "source-list-map": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
         "tapable": {
           "version": "1.1.3",
           "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
           "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA=="
-        },
-        "webpack-sources": {
-          "version": "1.4.3",
-          "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
-          "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-          "requires": {
-            "source-list-map": "^2.0.0",
-            "source-map": "~0.6.1"
-          }
         }
       }
     },
@@ -17605,7 +17529,6 @@
       "version": "1.4.3",
       "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
       "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-      "dev": true,
       "requires": {
         "source-list-map": "^2.0.0",
         "source-map": "~0.6.1"
@@ -17614,14 +17537,12 @@
         "source-list-map": {
           "version": "2.0.1",
           "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==",
+          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
-          "dev": true
         },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-          "dev": true
         }
       }
     },
@@ -17660,11 +17581,6 @@
         }
       }
     },
-    "whatwg-fetch": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz",
-      "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q=="
-    },
     "whatwg-mimetype": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz",
@@ -17780,9 +17696,9 @@
       "dev": true
     },
     "xtend": {
-      "version": "4.0.1",
+      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
-      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
     },
     "y18n": {
       "version": "4.0.0",

buildscripts/gen-ldflags.go

@@ -44,10 +44,21 @@ func releaseTag(version string) string {
 		relPrefix = prefix
 	}
 
+	relSuffix := ""
+	if hotfix := os.Getenv("MINIO_HOTFIX"); hotfix != "" {
+		relSuffix = hotfix
+	}
+
 	relTag := strings.Replace(version, " ", "-", -1)
 	relTag = strings.Replace(relTag, ":", "-", -1)
 	relTag = strings.Replace(relTag, ",", "", -1)
-	return relPrefix + "." + relTag
+	relTag = relPrefix + "." + relTag
+
+	if relSuffix != "" {
+		relTag += "." + relSuffix
+	}
+
+	return relTag
 }
 
 // commitID returns the abbreviated commit-id hash of the last commit.
@@ -68,5 +79,12 @@ func commitID() string {
 }
 
 func main() {
-	fmt.Println(genLDFlags(time.Now().UTC().Format(time.RFC3339)))
+	var version string
+	if len(os.Args) > 1 {
+		version = os.Args[1]
+	} else {
+		version = time.Now().UTC().Format(time.RFC3339)
+	}
+
+	fmt.Println(genLDFlags(version))
 }

buildscripts/verify-build.sh

@@ -56,28 +56,29 @@ function start_minio_erasure()
 
 function start_minio_erasure_sets()
 {
-    "${MINIO[@]}" server "${WORK_DIR}/erasure-disk-sets{1...32}" >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
+    export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
+    "${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 &
     sleep 15
 }
 
-function start_minio_zone_erasure_sets()
+function start_minio_pool_erasure_sets()
 {
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
-
+    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
+    "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
 
     sleep 40
 }
 
-function start_minio_zone_erasure_sets_ipv6()
+function start_minio_pool_erasure_sets_ipv6()
 {
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
-
+    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
-    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9001.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 
     sleep 40
 }
@@ -86,10 +87,10 @@ function start_minio_dist_erasure()
 {
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
+    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
+    for i in $(seq 0 3); do
-    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
+        "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
+    done
 
     sleep 40
 }
@@ -112,7 +113,8 @@ function run_test_fs()
     return "$rv"
 }
 
-function run_test_erasure_sets() {
+function run_test_erasure_sets()
+{
     start_minio_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
@@ -129,9 +131,9 @@ function run_test_erasure_sets() {
     return "$rv"
 }
 
-function run_test_zone_erasure_sets()
+function run_test_pool_erasure_sets()
 {
-    start_minio_zone_erasure_sets
+    start_minio_pool_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
@@ -142,20 +144,20 @@ function run_test_zone_erasure_sets()
     if [ "$rv" -ne 0 ]; then
         for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/zone-minio-900$i.log"
+            cat "$WORK_DIR/pool-minio-900$i.log"
         done
     fi
 
     for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/zone-minio-900$i.log"
+        rm -f "$WORK_DIR/pool-minio-900$i.log"
     done
 
     return "$rv"
 }
 
-function run_test_zone_erasure_sets_ipv6()
+function run_test_pool_erasure_sets_ipv6()
 {
-    start_minio_zone_erasure_sets_ipv6
+    start_minio_pool_erasure_sets_ipv6
 
     export SERVER_ENDPOINT="[::1]:9000"
 
@@ -168,12 +170,12 @@ function run_test_zone_erasure_sets_ipv6()
     if [ "$rv" -ne 0 ]; then
         for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/zone-minio-ipv6-900$i.log"
+            cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
         done
     fi
 
     for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/zone-minio-ipv6-900$i.log"
+        rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
     done
 
     return "$rv"
@@ -220,7 +222,7 @@ function run_test_dist_erasure()
 
     rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
 
-   return "$rv"
+    return "$rv"
 }
 
 function purge()
@@ -293,14 +295,14 @@ function main()
     fi
 
     echo "Testing in Distributed Eraure expanded setup"
-    if ! run_test_zone_erasure_sets; then
+    if ! run_test_pool_erasure_sets; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1
     fi
 
     echo "Testing in Distributed Erasure expanded setup with ipv6"
-    if ! run_test_zone_erasure_sets_ipv6; then
+    if ! run_test_pool_erasure_sets_ipv6; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1

buildscripts/verify-healing.sh

@@ -29,44 +29,52 @@ MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
 
 function start_minio_3_node() {
-    declare -a minio_pids
-    declare -a ARGS
     export MINIO_ACCESS_KEY=minio
     export MINIO_SECRET_KEY=minio123
     export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
     start_port=$(shuf -i 10000-65000 -n 1)
+    args=""
     for i in $(seq 1 3); do
-        ARGS+=("http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/")
+        args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
-    "${MINIO[@]}" --address ":$[$start_port+1]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+    "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    minio_pids[0]=$!
+    disown $!
-    disown "${minio_pids[0]}"
 
-    "${MINIO[@]}" --address ":$[$start_port+2]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+    "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    minio_pids[1]=$!
+    disown $!
-    disown "${minio_pids[1]}"
 
-    "${MINIO[@]}" --address ":$[$start_port+3]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+    "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    minio_pids[2]=$!
+    disown $!
-    disown "${minio_pids[2]}"
 
     sleep "$1"
-    for pid in "${minio_pids[@]}"; do
+    if [ "$(pgrep -c minio)" -ne 3 ]; then
-        if ! kill "$pid"; then
+        for i in $(seq 1 3); do
-            for i in $(seq 1 3); do
+            echo "server$i log:"
-                echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
-                cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
-            done
+        echo "FAILED"
-            echo "FAILED"
+        purge "$WORK_DIR"
-            purge "$WORK_DIR"
+        exit 1
-            exit 1
+    fi
-        fi
+    if ! pkill minio; then
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+
+    sleep 1;
+    if pgrep minio; then
         # forcibly killing, to proceed further properly.
-        kill -9 "$pid"
+        if ! pkill -9 minio; then
-        sleep 1 # wait 1sec per pid
+            echo "no minio process running anymore, proceed."
-    done
+        fi
+    fi
 }
 
 

cmd/admin-bucket-handlers.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -128,6 +127,12 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
+	update := r.URL.Query().Get("update") == "true"
+
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
@@ -135,10 +140,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 
 	// Check if bucket exists.
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
@@ -163,23 +164,23 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	host, port, err := net.SplitHostPort(target.Endpoint)
+
-	if err != nil {
+	sameTarget, _ := isLocalHost(target.URL().Hostname(), target.URL().Port(), globalMinioPort)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-	sameTarget, _ := isLocalHost(host, port, globalMinioPort)
 	if sameTarget && bucket == target.TargetBucket {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
 		return
 	}
+
 	target.SourceBucket = bucket
-	target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+	if !update {
+		target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+	}
 	if target.Arn == "" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target); err != nil {
+
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -187,14 +188,12 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
-
 	}
 	tgtBytes, err := json.Marshal(&targets)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-
 	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -218,7 +217,10 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arnType := vars["type"]
-
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketTargetAction)
 	if objectAPI == nil {
@@ -255,16 +257,16 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 	arn := vars["arn"]
 
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if !globalIsErasure {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 
 	// Check if bucket exists.
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {

cmd/admin-handlers-config-kv.go

@@ -42,14 +42,14 @@ import (
 
 func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return auth.Credentials{}, nil
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
+	cred, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return cred, nil
@@ -130,8 +130,8 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
+	dynamic, err := cfg.ReadConfig(bytes.NewReader(kvBytes))
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -158,6 +158,17 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
+	// Apply dynamic values.
+	if err := applyDynamicConfig(GlobalContext, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	globalNotificationSys.SignalService(serviceReloadDynamic)
+
+	// If all values were dynamic, tell the client.
+	if dynamic {
+		w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
+	}
 	writeSuccessResponseHeadersOnly(w)
 }
 
@@ -266,7 +277,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -378,7 +389,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	cfg := newServerConfig()
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"path"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -35,14 +36,14 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	var adminAPIErr APIErrorCode
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return nil, cred
@@ -129,13 +130,40 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
+	vars := mux.Vars(r)
-	if objectAPI == nil {
+	name := vars["accessKey"]
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
-	name := vars["accessKey"]
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	accessKey := cred.AccessKey
+	if cred.ParentUser != "" {
+		accessKey = cred.ParentUser
+	}
+
+	implicitPerm := name == accessKey
+	if !implicitPerm {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     accessKey,
+			Action:          iampolicy.GetUserAdminAction,
+			ConditionValues: getConditionValues(r, "", accessKey, claims),
+			IsOwner:         owner,
+			Claims:          claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	}
 
 	userInfo, err := globalIAMSys.GetUserInfo(name)
 	if err != nil {
@@ -304,7 +332,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
 
-	// Custom IAM policies not allowed for admin user.
+	// This API is not allowed to lookup accessKey user status
 	if accessKey == globalActiveCred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
@@ -330,20 +358,53 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
 
-	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
+	vars := mux.Vars(r)
-	if objectAPI == nil {
+	accessKey := path.Clean(vars["accessKey"])
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
-	accessKey := vars["accessKey"]
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
 
-	// Custom IAM policies not allowed for admin user.
+	// Not allowed to add a user with same access key as root credential
-	if accessKey == globalActiveCred.AccessKey {
+	if owner && accessKey == cred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
 		return
 	}
 
+	if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
+		// Incoming access key matches parent user then we should
+		// reject password change requests.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
+	implicitPerm := accessKey == cred.AccessKey
+	if !implicitPerm {
+		parentUser := cred.ParentUser
+		if parentUser == "" {
+			parentUser = cred.AccessKey
+		}
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     parentUser,
+			Action:          iampolicy.CreateUserAdminAction,
+			ConditionValues: getConditionValues(r, "", parentUser, claims),
+			IsOwner:         owner,
+			Claims:          claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	}
+
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
@@ -386,8 +447,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -398,6 +459,12 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
+		return
+	}
+
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
@@ -411,12 +478,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Disallow creating service accounts by root user.
-	if owner {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
-		return
-	}
-
 	parentUser := cred.AccessKey
 	if cred.ParentUser != "" {
 		parentUser = cred.ParentUser
@@ -465,8 +526,8 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -520,8 +581,8 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -572,15 +633,15 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
-// AccountUsageInfoHandler returns usage
+// AccountInfoHandler returns usage
-func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AccountUsageInfo")
+	ctx := newContext(r, w, "AccountInfo")
 
-	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "AccountInfo", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -645,8 +706,16 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 		accountName = cred.ParentUser
 	}
 
-	acctInfo := madmin.AccountUsageInfo{
+	policies, err := globalIAMSys.PolicyDBGet(accountName, false)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	acctInfo := madmin.AccountInfo{
 		AccountName: accountName,
+		Policy:      globalIAMSys.GetCombinedPolicy(policies...),
 	}
 
 	for _, bucket := range buckets {

cmd/admin-handlers.go

@@ -35,13 +35,13 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bandwidth"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -86,8 +86,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalInplaceUpdateDisabled {
-		// if MINIO_UPDATE=off - inplace update is disabled, mostly
+		// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
-		// in containers.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
 	}
@@ -128,7 +127,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if lrTime.Sub(crTime) < 0 {
+	if lrTime.Sub(crTime) <= 0 {
 		updateStatus := madmin.ServerUpdateStatus{
 			CurrentVersion: Version,
 			UpdatedVersion: Version,
@@ -206,9 +205,10 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	var objectAPI ObjectLayer
-	if serviceSig == serviceRestart {
+	switch serviceSig {
+	case serviceRestart:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
-	} else {
+	case serviceStop:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
 	}
 	if objectAPI == nil {
@@ -349,7 +349,7 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquorum int) *madmin.LockEntry {
+func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
 		Resource:   resource,
@@ -357,40 +357,34 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquor
 		Source:     l.Source,
 		Owner:      l.Owner,
 		ID:         l.UID,
+		Quorum:     l.Quorum,
 	}
 	if l.Writer {
 		entry.Type = "WRITE"
-		entry.Quorum = wquorum
 	} else {
 		entry.Type = "READ"
-		entry.Quorum = rquorum
 	}
 	return entry
 }
 
-func topLockEntries(peerLocks []*PeerLocks, count int, rquorum, wquorum int, stale bool) madmin.LockEntries {
+func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
 			continue
 		}
-		for _, locks := range peerLock.Locks {
+		for k, v := range peerLock.Locks {
-			for k, v := range locks {
+			for _, lockReqInfo := range v {
-				for _, lockReqInfo := range v {
+				if val, ok := entryMap[lockReqInfo.UID]; ok {
-					if val, ok := entryMap[lockReqInfo.UID]; ok {
+					val.ServerList = append(val.ServerList, peerLock.Addr)
-						val.ServerList = append(val.ServerList, peerLock.Addr)
+				} else {
-					} else {
+					entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
-						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr, rquorum, wquorum)
-					}
 				}
 			}
 		}
 	}
 	var lockEntries madmin.LockEntries
 	for _, v := range entryMap {
-		if len(lockEntries) == count {
-			break
-		}
 		if stale {
 			lockEntries = append(lockEntries, *v)
 			continue
@@ -406,7 +400,7 @@ func topLockEntries(peerLocks []*PeerLocks, count int, rquorum, wquorum int, sta
 // PeerLocks holds server information result of one node
 type PeerLocks struct {
 	Addr  string
-	Locks GetLocksResp
+	Locks map[string][]lockRequesterInfo
 }
 
 // TopLocksHandler Get list of locks in use
@@ -433,12 +427,13 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 
 	peerLocks := globalNotificationSys.GetLocks(ctx, r)
 
-	rquorum := getReadQuorum(objectAPI.SetDriveCount())
+	topLocks := topLockEntries(peerLocks, stale)
-	wquorum := getWriteQuorum(objectAPI.SetDriveCount())
 
-	topLocks := topLockEntries(peerLocks, count, rquorum, wquorum, stale)
+	// Marshal API response upto requested count.
+	if len(topLocks) > count && count > 0 {
+		topLocks = topLocks[:count]
+	}
 
-	// Marshal API response
 	jsonBytes, err := json.Marshal(topLocks)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -466,8 +461,15 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 
 	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
+	// Validate request signature.
-	if objectAPI == nil {
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -564,8 +566,15 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 
 	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
+	// Validate request signature.
-	if objectAPI == nil {
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -799,7 +808,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	case hip.clientToken == "":
 		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
 		go func() {
-			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
+			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh, objectAPI)
 			hr := healResp{respBytes, apiErr, errMsg}
 			respCh <- hr
 		}()
@@ -898,14 +907,14 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	var cred auth.Credentials
 	var adminAPIErr APIErrorCode
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return nil, cred
@@ -1011,6 +1020,15 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 	if !ok {
 		return false
 	}
+
+	// Handle browser requests separately filter them and return.
+	if HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+"/upload") {
+		if errOnly {
+			return trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
+		}
+		return true
+	}
+
 	trace := trcAll || !HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator)
 	if errOnly {
 		return trace && trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
@@ -1028,7 +1046,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1040,7 +1058,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
@@ -1108,7 +1126,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	logCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
@@ -1240,26 +1258,26 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-// OBDInfoHandler - GET /minio/admin/v3/obdinfo
+// HealthInfoHandler - GET /minio/admin/v3/healthinfo
 // ----------
-// Get server on-board diagnostics
+// Get server health info
-func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "OBDInfo")
+	ctx := newContext(r, w, "HealthInfo")
 
-	defer logger.AuditLog(w, r, "OBDInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "HealthInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.OBDInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	query := r.URL.Query()
-	obdInfo := madmin.OBDInfo{}
+	healthInfo := madmin.HealthInfo{}
-	obdInfoCh := make(chan madmin.OBDInfo)
+	healthInfoCh := make(chan madmin.HealthInfo)
 
 	enc := json.NewEncoder(w)
-	partialWrite := func(oinfo madmin.OBDInfo) {
+	partialWrite := func(oinfo madmin.HealthInfo) {
-		obdInfoCh <- oinfo
+		healthInfoCh <- oinfo
 	}
 
 	setCommonHeaders(w)
@@ -1272,8 +1290,8 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
 			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 		encodedErrorResponse := encodeResponse(errorResponse)
-		obdInfo.Error = string(encodedErrorResponse)
+		healthInfo.Error = string(encodedErrorResponse)
-		logger.LogIf(ctx, enc.Encode(obdInfo))
+		logger.LogIf(ctx, enc.Encode(healthInfo))
 	}
 
 	deadline := 3600 * time.Second
@@ -1289,113 +1307,107 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	if err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
 	defer nsLock.Unlock()
 
 	go func() {
-		defer close(obdInfoCh)
+		defer close(healthInfoCh)
-
-		if log := query.Get("log"); log == "true" {
-			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
-			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
-		}
 
 		if cpu := query.Get("syscpu"); cpu == "true" {
-			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
+			cpuInfo := getLocalCPUInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo)
-			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, globalNotificationSys.CPUInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
-			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
+			diskHwInfo := getLocalDiskHwInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
+			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, diskHwInfo)
-			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
+			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
-			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
+			osInfo := getLocalOsInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
+			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, osInfo)
-			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
+			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, globalNotificationSys.OsInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if mem := query.Get("sysmem"); mem == "true" {
-			memInfo := getLocalMemOBD(deadlinedCtx, r)
+			memInfo := getLocalMemInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, memInfo)
-			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, globalNotificationSys.MemInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if proc := query.Get("sysprocess"); proc == "true" {
-			procInfo := getLocalProcOBD(deadlinedCtx, r)
+			procInfo := getLocalProcInfo(deadlinedCtx, r)
 
-			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, procInfo)
-			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, globalNotificationSys.ProcInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if config := query.Get("minioconfig"); config == "true" {
 			cfg, err := readServerConfig(ctx, objectAPI)
 			logger.LogIf(ctx, err)
-			obdInfo.Minio.Config = cfg
+			healthInfo.Minio.Config = cfg
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if drive := query.Get("perfdrive"); drive == "true" {
-			// Get drive obd details from local server's drive(s)
+			// Get drive perf details from local server's drive(s)
-			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
+			drivePerfSerial := getLocalDrives(deadlinedCtx, false, globalEndpoints, r)
-			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
+			drivePerfParallel := getLocalDrives(deadlinedCtx, true, globalEndpoints, r)
 
 			errStr := ""
-			if driveOBDSerial.Error != "" {
+			if drivePerfSerial.Error != "" {
-				errStr = "serial: " + driveOBDSerial.Error
+				errStr = "serial: " + drivePerfSerial.Error
 			}
-			if driveOBDParallel.Error != "" {
+			if drivePerfParallel.Error != "" {
-				errStr = errStr + " parallel: " + driveOBDParallel.Error
+				errStr = errStr + " parallel: " + drivePerfParallel.Error
 			}
 
-			driveOBD := madmin.ServerDrivesOBDInfo{
+			driveInfo := madmin.ServerDrivesInfo{
-				Addr:     driveOBDSerial.Addr,
+				Addr:     drivePerfSerial.Addr,
-				Serial:   driveOBDSerial.Serial,
+				Serial:   drivePerfSerial.Serial,
-				Parallel: driveOBDParallel.Parallel,
+				Parallel: drivePerfParallel.Parallel,
 				Error:    errStr,
 			}
-			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
+			healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, driveInfo)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 
-			// Notify all other MinIO peers to report drive obd numbers
+			// Notify all other MinIO peers to report drive perf numbers
-			driveOBDs := globalNotificationSys.DriveOBDInfoChan(deadlinedCtx)
+			driveInfos := globalNotificationSys.DrivePerfInfoChan(deadlinedCtx)
-			for obd := range driveOBDs {
+			for obd := range driveInfos {
-				obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, obd)
+				healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, obd)
-				partialWrite(obdInfo)
+				partialWrite(healthInfo)
 			}
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
-			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
+			healthInfo.Perf.Net = append(healthInfo.Perf.Net, globalNotificationSys.NetInfo(deadlinedCtx))
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 
-			netOBDs := globalNotificationSys.DispatchNetOBDChan(deadlinedCtx)
+			netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
-			for obd := range netOBDs {
+			for netInfo := range netInfos {
-				obdInfo.Perf.Net = append(obdInfo.Perf.Net, obd)
+				healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo)
-				partialWrite(obdInfo)
+				partialWrite(healthInfo)
 			}
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 
-			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
+			healthInfo.Perf.NetParallel = globalNotificationSys.NetPerfParallelInfo(deadlinedCtx)
-			partialWrite(obdInfo)
+			partialWrite(healthInfo)
 		}
 
 	}()
@@ -1405,7 +1417,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 
 	for {
 		select {
-		case oinfo, ok := <-obdInfoCh:
+		case oinfo, ok := <-healthInfoCh:
 			if !ok {
 				return
 			}
@@ -1424,6 +1436,59 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 
 }
 
+// BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
+// ----------
+// Get bandwidth consumption information
+func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "BandwidthMonitor")
+
+	defer logger.AuditLog(w, r, "BandwidthMonitor", mustGetClaimsFromToken(r))
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	setEventStreamHeaders(w)
+	reportCh := make(chan bandwidth.Report, 1)
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+	bucketsRequestedString := r.URL.Query().Get("buckets")
+	bucketsRequested := strings.Split(bucketsRequestedString, ",")
+	go func() {
+		for {
+			reportCh <- globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				time.Sleep(2 * time.Second)
+			}
+		}
+	}()
+	for {
+		select {
+		case report := <-reportCh:
+			enc := json.NewEncoder(w)
+			err := enc.Encode(report)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-keepAliveTicker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 // ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information
@@ -1432,22 +1497,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
+	// Validate request signature.
-	if objectAPI == nil {
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
 
-	buckets := madmin.Buckets{}
-	objects := madmin.Objects{}
-	usage := madmin.Usage{}
-
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err == nil {
-		buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
-		objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
-		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
-	}
-
 	vault := fetchVaultStatus()
 
 	ldap := madmin.LDAP{}
@@ -1469,35 +1525,54 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
 
-	// Fetching the Storage information, ignore any errors.
-	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
-
-	var backend interface{}
-	if storageInfo.Backend.Type == BackendType(madmin.Erasure) {
-		backend = madmin.ErasureBackend{
-			Type:             madmin.ErasureType,
-			OnlineDisks:      storageInfo.Backend.OnlineDisks.Sum(),
-			OfflineDisks:     storageInfo.Backend.OfflineDisks.Sum(),
-			StandardSCData:   storageInfo.Backend.StandardSCData,
-			StandardSCParity: storageInfo.Backend.StandardSCParity,
-			RRSCData:         storageInfo.Backend.RRSCData,
-			RRSCParity:       storageInfo.Backend.RRSCParity,
-		}
-	} else {
-		backend = madmin.FSBackend{
-			Type: madmin.FsType,
-		}
-	}
-
-	mode := "safemode"
-	if newObjectLayerFn() != nil {
-		mode = "online"
-	}
-
 	server := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
 	servers = append(servers, server)
 
+	var backend interface{}
+	mode := madmin.ObjectLayerInitializing
+
+	buckets := madmin.Buckets{}
+	objects := madmin.Objects{}
+	usage := madmin.Usage{}
+
+	objectAPI := newObjectLayerFn()
+	if objectAPI != nil {
+		mode = madmin.ObjectLayerOnline
+		// Load data usage
+		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+		if err == nil {
+			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
+			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
+			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
+		}
+
+		// Fetching the backend information
+		backendInfo := objectAPI.BackendInfo()
+		if backendInfo.Type == BackendType(madmin.Erasure) {
+			// Calculate the number of online/offline disks of all nodes
+			var allDisks []madmin.Disk
+			for _, s := range servers {
+				allDisks = append(allDisks, s.Disks...)
+			}
+			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+
+			backend = madmin.ErasureBackend{
+				Type:             madmin.ErasureType,
+				OnlineDisks:      onlineDisks.Sum(),
+				OfflineDisks:     offlineDisks.Sum(),
+				StandardSCData:   backendInfo.StandardSCData,
+				StandardSCParity: backendInfo.StandardSCParity,
+				RRSCData:         backendInfo.RRSCData,
+				RRSCParity:       backendInfo.RRSCParity,
+			}
+		} else {
+			backend = madmin.FSBackend{
+				Type: madmin.FsType,
+			}
+		}
+	}
+
 	domain := globalDomainNames
 	services := madmin.Services{
 		Vault:         vault,
@@ -1507,25 +1582,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
-	// find all disks which belong to each respective endpoints
-	for i := range servers {
-		for _, disk := range storageInfo.Disks {
-			if strings.Contains(disk.Endpoint, servers[i].Endpoint) {
-				servers[i].Disks = append(servers[i].Disks, disk)
-			}
-		}
-	}
-
-	// add all the disks local to this server.
-	for _, disk := range storageInfo.Disks {
-		if disk.DrivePath == "" && disk.Endpoint == "" {
-			continue
-		}
-		if disk.Endpoint == disk.DrivePath {
-			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
-		}
-	}
-
 	infoMsg := madmin.InfoMessage{
 		Mode:         mode,
 		Domain:       domain,

cmd/admin-handlers_test.go

@@ -105,8 +105,8 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureZones{zones: make([]*erasureSets, 1)}
+	objLayer := &erasureServerPools{serverPools: make([]*erasureSets, 1)}
-	objLayer.zones[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer.serverPools[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}

cmd/admin-heal-ops.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -93,15 +95,15 @@ type allHealState struct {
 }
 
 // newHealState - initialize global heal state management
-func newHealState() *allHealState {
+func newHealState(cleanup bool) *allHealState {
-	healState := &allHealState{
+	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
 		healLocalDisks: map[Endpoint]struct{}{},
 	}
-
+	if cleanup {
-	go healState.periodicHealSeqsClean(GlobalContext)
+		go hstate.periodicHealSeqsClean(GlobalContext)
-
+	}
-	return healState
+	return hstate
 }
 
 func (ahs *allHealState) healDriveCount() int {
@@ -143,9 +145,13 @@ func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
+	periodicTimer := time.NewTimer(time.Minute * 5)
+	defer periodicTimer.Stop()
+
 	for {
 		select {
-		case <-time.After(time.Minute * 5):
+		case <-periodicTimer.C:
+			periodicTimer.Reset(time.Minute * 5)
 			now := UTCNow()
 			ahs.Lock()
 			for path, h := range ahs.healSeqMap {
@@ -228,7 +234,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 // `keepHealSeqStateDuration`. This function also launches a
 // background routine to clean up heal results after the
 // aforementioned duration.
-func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
+func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLayer) (
 	respBytes []byte, apiErr APIError, errMsg string) {
 
 	if h.forceStarted {
@@ -265,7 +271,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	ahs.healSeqMap[hpath] = h
 
 	// Launch top-level background heal go-routine
-	go h.healSequenceStart()
+	go h.healSequenceStart(objAPI)
 
 	clientToken := h.clientToken
 	if globalIsDistErasure {
@@ -609,7 +615,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 // routine for completion, and (2) listens for external stop
 // signals. When either event happens, it sets the finish status for
 // the heal-sequence.
-func (h *healSequence) healSequenceStart() {
+func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	// Set status as running
 	h.mutex.Lock()
 	h.currentStatus.Summary = healRunningStatus
@@ -617,7 +623,7 @@ func (h *healSequence) healSequenceStart() {
 	h.mutex.Unlock()
 
 	if h.sourceCh == nil {
-		go h.traverseAndHeal()
+		go h.traverseAndHeal(objAPI)
 	} else {
 		go h.healFromSourceCh()
 	}
@@ -657,6 +663,10 @@ func (h *healSequence) healSequenceStart() {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
+	globalHealConfigMu.Lock()
+	opts := globalHealConfig
+	globalHealConfigMu.Unlock()
+
 	// Send heal request
 	task := healTask{
 		bucket:     source.bucket,
@@ -668,6 +678,12 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
+	if opts.Bitrot {
+		task.opts.ScanMode = madmin.HealDeepScan
+	}
+
+	// Wait and proceed if there are active requests
+	waitForLowHTTPReq(opts.IOCount, opts.Sleep)
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++
@@ -766,28 +782,28 @@ func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
 }
 
-func (h *healSequence) healDiskMeta() error {
+func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
-	// Start with format healing
+	// Try to pro-actively heal backend-encrypted file.
-	if err := h.healDiskFormat(); err != nil {
+	if err := h.queueHealTask(healSource{
-		return err
+		bucket: minioMetaBucket,
+		object: backendEncryptedFile,
+	}, madmin.HealItemBucketMetadata); err != nil {
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
 	}
 
 	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
-		return err
-	}
-
-	// Start healing the bucket config prefix.
-	return h.healMinioSysMeta(bucketConfigPrefix)()
 }
 
-func (h *healSequence) healItems(bucketsOnly bool) error {
+func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
-	if err := h.healDiskMeta(); err != nil {
+	if err := h.healDiskMeta(objAPI); err != nil {
 		return err
 	}
 
 	// Heal buckets and objects
-	return h.healBuckets(bucketsOnly)
+	return h.healBuckets(objAPI, bucketsOnly)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -797,30 +813,29 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 // quit signal is received, this routine cannot quit immediately and
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
-func (h *healSequence) traverseAndHeal() {
+func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
 	bucketsOnly := false // Heals buckets and objects also.
-	h.traverseAndHealDoneCh <- h.healItems(bucketsOnly)
+	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
 	close(h.traverseAndHealDoneCh)
 }
 
 // healMinioSysMeta - heals all files under a given meta prefix, returns a function
 // which in-turn heals the respective meta directory path and any files in int.
-func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
+func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) func() error {
 	return func() error {
-		// Get current object layer instance.
-		objectAPI := newObjectLayerWithoutSafeModeFn()
-		if objectAPI == nil {
-			return errServerNotInitialized
-		}
-
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
 
+			// Skip metacache entries healing
+			if strings.HasPrefix(object, "buckets/.minio.sys/.metacache/") {
+				return nil
+			}
+
 			err := h.queueHealTask(healSource{
 				bucket:    bucket,
 				object:    object,
@@ -843,39 +858,32 @@ func (h *healSequence) healDiskFormat() error {
 		return errHealStopSignalled
 	}
 
-	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets(bucketsOnly bool) error {
+func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(h.bucket, bucketsOnly)
+		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	// Get current object layer instance.
+	buckets, err := objAPI.ListBuckets(h.ctx)
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	buckets, err := objectAPI.ListBucketsHeal(h.ctx)
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
+	// Heal latest buckets first.
+	sort.Slice(buckets, func(i, j int) bool {
+		return buckets[i].Created.After(buckets[j].Created)
+	})
+
 	for _, bucket := range buckets {
-		if err = h.healBucket(bucket.Name, bucketsOnly); err != nil {
+		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
 			return err
 		}
 	}
@@ -884,13 +892,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 }
 
 // healBucket - traverses and heals given bucket
-func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
+func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly bool) error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 			return err
@@ -905,7 +907,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		if h.object != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			oi, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
+			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -919,7 +921,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
+	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
 		// Object might have been deleted, by the time heal
 		// was attempted we ignore this object an move on.
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -931,12 +933,6 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object, versionID string) error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-router.go

@@ -116,7 +116,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 			// Add user IAM
 
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountUsageInfoHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
 
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
@@ -180,19 +180,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 				// PutBucketQuotaConfig
 				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
 					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-			}
-			// Bucket replication operations
-			// GetBucketTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
-		}
 
+				// Bucket replication operations
+				// GetBucketTargetHandler
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+					httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+				// RemoveRemoteTargetHandler
+				adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+			}
+		}
 		// -- Top APIs --
 		// Top locks
 		if globalIsDistErasure {
@@ -211,13 +211,18 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 
 		if !globalIsGateway {
-			// -- OBD API --
+			// Keep obdinfo for backward compatibility with mc
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
+				HandlerFunc(httpTraceHdrs(adminAPI.HealthInfoHandler))
+			// -- Health API --
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.HealthInfoHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
+				HandlerFunc(httpTraceHdrs(adminAPI.BandwidthMonitorHandler))
 		}
 	}
 
 	// If none of the routes match add default error handler routes
 	adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler)
-	adminRouter.MethodNotAllowedHandler = httpTraceAll(errorResponseHandler)
+	adminRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("Admin"))
 }

cmd/admin-server-info.go

@@ -24,13 +24,14 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
+	var localEndpoints Endpoints
 	addr := r.Host
 	if globalIsDistErasure {
-		addr = GetLocalPeer(endpointZones)
+		addr = GetLocalPeer(endpointServerPools)
 	}
 	network := make(map[string]string)
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
 			nodeName := endpoint.Host
 			if nodeName == "" {
@@ -39,6 +40,7 @@ func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin
 			if endpoint.IsLocal {
 				// Only proceed for local endpoints
 				network[nodeName] = "online"
+				localEndpoints = append(localEndpoints, endpoint)
 				continue
 			}
 			_, present := network[nodeName]
@@ -52,6 +54,11 @@ func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin
 		}
 	}
 
+	localDisks, _ := initStorageDisksWithErrors(localEndpoints)
+	defer closeStorageDisks(localDisks)
+
+	storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings())
+
 	return madmin.ServerProperties{
 		State:    "ok",
 		Endpoint: addr,
@@ -59,5 +66,6 @@ func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin
 		Version:  Version,
 		CommitID: CommitID,
 		Network:  network,
+		Disks:    storageInfo.Disks,
 	}
 }

cmd/api-datatypes.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/xml"
+	"time"
 )
 
 // DeletedObject objects deleted
@@ -26,12 +27,44 @@ type DeletedObject struct {
 	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
 	ObjectName            string `xml:"Key,omitempty"`
 	VersionID             string `xml:"VersionId,omitempty"`
+
+	// MinIO extensions to support delete marker replication
+	// Replication status of DeleteMarker
+	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus,omitempty"`
+	// MTime of DeleteMarker on source that needs to be propagated to replica
+	DeleteMarkerMTime DeleteMarkerMTime `xml:"DeleteMarkerMTime,omitempty"`
+	// Status of versioned delete (of object or DeleteMarker)
+	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus,omitempty"`
+	// PurgeTransitioned is nonempty if object is in transition tier
+	PurgeTransitioned string `xml:"PurgeTransitioned,omitempty"`
+}
+
+// DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
+type DeleteMarkerMTime struct {
+	time.Time
+}
+
+// MarshalXML encodes expiration date if it is non-zero and encodes
+// empty string otherwise
+func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if t.Time.IsZero() {
+		return nil
+	}
+	return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
 }
 
 // ObjectToDelete carries key name for the object to delete.
 type ObjectToDelete struct {
 	ObjectName string `xml:"Key"`
 	VersionID  string `xml:"VersionId"`
+	// Replication status of DeleteMarker
+	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
+	// Status of versioned delete (of object or DeleteMarker)
+	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus"`
+	// Version ID of delete marker
+	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId"`
+	// PurgeTransitioned is nonempty if object is in transition tier
+	PurgeTransitioned string `xml:"PurgeTransitioned"`
 }
 
 // createBucketConfiguration container for bucket configuration request from client.

cmd/api-errors.go

@@ -106,22 +106,25 @@ const (
 	ErrNoSuchCORSConfiguration
 	ErrNoSuchWebsiteConfiguration
 	ErrReplicationConfigurationNotFoundError
-	ErrReplicationDestinationNotFoundError
+	ErrRemoteDestinationNotFoundError
 	ErrReplicationDestinationMissingLock
-	ErrReplicationTargetNotFoundError
+	ErrRemoteTargetNotFoundError
 	ErrReplicationRemoteConnectionError
 	ErrBucketRemoteIdenticalToSource
 	ErrBucketRemoteAlreadyExists
+	ErrBucketRemoteLabelInUse
 	ErrBucketRemoteArnTypeInvalid
 	ErrBucketRemoteArnInvalid
 	ErrBucketRemoteRemoveDisallowed
-	ErrReplicationTargetNotVersionedError
+	ErrRemoteTargetNotVersionedError
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrBucketReplicationDisabledError
+	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
+	ErrInvalidVersionID
 	ErrNoSuchVersion
 	ErrNotImplemented
 	ErrPreconditionFailed
@@ -232,6 +235,7 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
+	ErrClientDisconnected
 	ErrOperationMaxedOut
 	ErrInvalidRequest
 	// MinIO storage class error codes
@@ -360,6 +364,7 @@ const (
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
 	ErrAdminAccountNotEligible
+	ErrAccountNotEligible
 	ErrServiceAccountNotFound
 	ErrPostPolicyConditionInvalidFormat
 )
@@ -554,11 +559,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrNoSuchVersion: {
+	ErrInvalidVersionID: {
 		Code:           "InvalidArgument",
 		Description:    "Invalid version id specified",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrNoSuchVersion: {
+		Code:           "NoSuchVersion",
+		Description:    "The specified version does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
 		Description:    "A header you provided implies functionality that is not implemented",
@@ -809,9 +819,9 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication configuration was not found",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrReplicationDestinationNotFoundError: {
+	ErrRemoteDestinationNotFoundError: {
-		Code:           "ReplicationDestinationNotFoundError",
+		Code:           "RemoteDestinationNotFoundError",
-		Description:    "The replication destination bucket does not exist",
+		Description:    "The remote destination bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationDestinationMissingLock: {
@@ -819,29 +829,34 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication destination bucket does not have object locking enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationTargetNotFoundError: {
+	ErrRemoteTargetNotFoundError: {
-		Code:           "XminioAdminReplicationTargetNotFoundError",
+		Code:           "XMinioAdminRemoteTargetNotFoundError",
-		Description:    "The replication target does not exist",
+		Description:    "The remote target does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationRemoteConnectionError: {
-		Code:           "XminioAdminReplicationRemoteConnectionError",
+		Code:           "XMinioAdminReplicationRemoteConnectionError",
-		Description:    "Remote service endpoint or target bucket not available",
+		Description:    "Remote service connection error - please check remote service credentials and target bucket",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrBucketRemoteIdenticalToSource: {
-		Code:           "XminioAdminRemoteIdenticalToSource",
+		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteAlreadyExists: {
-		Code:           "XminioAdminBucketRemoteAlreadyExists",
+		Code:           "XMinioAdminBucketRemoteAlreadyExists",
 		Description:    "The remote target already exists",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrBucketRemoteLabelInUse: {
+		Code:           "XMinioAdminBucketRemoteLabelInUse",
+		Description:    "The remote target with this label already exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketRemoteRemoveDisallowed: {
 		Code:           "XMinioAdminRemoteRemoveDisallowed",
-		Description:    "Replication configuration exists with this ARN.",
+		Description:    "This ARN is in use by an existing configuration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteArnTypeInvalid: {
@@ -854,9 +869,9 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket remote ARN does not have correct format",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationTargetNotVersionedError: {
+	ErrRemoteTargetNotVersionedError: {
-		Code:           "ReplicationTargetNotVersionedError",
+		Code:           "RemoteTargetNotVersionedError",
-		Description:    "The replication target does not have versioning enabled",
+		Description:    "The remote target does not have versioning enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationSourceNotVersionedError: {
@@ -909,6 +924,11 @@ var errorCodes = errorCodeMap{
 		Description:    "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectRestoreAlreadyInProgress: {
+		Code:           "RestoreAlreadyInProgress",
+		Description:    "Object restore is already in progress",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	/// Bucket notification related errors.
 	ErrEventNotification: {
 		Code:           "InvalidArgument",
@@ -1210,6 +1230,11 @@ var errorCodes = errorCodeMap{
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrClientDisconnected: {
+		Code:           "ClientDisconnected",
+		Description:    "Client disconnected before response was ready",
+		HTTPStatusCode: 499, // No official code, use nginx value.
+	},
 	ErrOperationMaxedOut: {
 		Code:           "SlowDown",
 		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
@@ -1708,12 +1733,17 @@ var errorCodes = errorCodeMap{
 	ErrAddUserInvalidArgument: {
 		Code:           "XMinioInvalidIAMCredentials",
 		Description:    "User is not allowed to be same as admin access key",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrAdminAccountNotEligible: {
 		Code:           "XMinioInvalidIAMCredentials",
 		Description:    "The administrator key is not eligible for this operation",
-		HTTPStatusCode: http.StatusConflict,
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrAccountNotEligible: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "The account key is not eligible for this operation",
+		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrServiceAccountNotFound: {
 		Code:           "XMinioInvalidIAMCredentials",
@@ -1736,6 +1766,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Only return ErrClientDisconnected if the provided context is actually canceled.
+	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	select {
+	case <-ctx.Done():
+		if ctx.Err() == context.Canceled {
+			return ErrClientDisconnected
+		}
+	default:
+	}
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -1852,6 +1892,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchKey
 	case MethodNotAllowed:
 		apiErr = ErrMethodNotAllowed
+	case InvalidVersionID:
+		apiErr = ErrInvalidVersionID
 	case VersionNotFound:
 		apiErr = ErrNoSuchVersion
 	case ObjectAlreadyExists:
@@ -1906,24 +1948,26 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchQuotaConfiguration
 	case BucketReplicationConfigNotFound:
 		apiErr = ErrReplicationConfigurationNotFoundError
-	case BucketReplicationDestinationNotFound:
+	case BucketRemoteDestinationNotFound:
-		apiErr = ErrReplicationDestinationNotFoundError
+		apiErr = ErrRemoteDestinationNotFoundError
 	case BucketReplicationDestinationMissingLock:
 		apiErr = ErrReplicationDestinationMissingLock
 	case BucketRemoteTargetNotFound:
-		apiErr = ErrReplicationTargetNotFoundError
+		apiErr = ErrRemoteTargetNotFoundError
 	case BucketRemoteConnectionErr:
 		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists
+	case BucketRemoteLabelInUse:
+		apiErr = ErrBucketRemoteLabelInUse
 	case BucketRemoteArnTypeInvalid:
 		apiErr = ErrBucketRemoteArnTypeInvalid
 	case BucketRemoteArnInvalid:
 		apiErr = ErrBucketRemoteArnInvalid
 	case BucketRemoteRemoveDisallowed:
 		apiErr = ErrBucketRemoteRemoveDisallowed
-	case BucketReplicationTargetNotVersioned:
+	case BucketRemoteTargetNotVersioned:
-		apiErr = ErrReplicationTargetNotVersionedError
+		apiErr = ErrRemoteTargetNotVersionedError
 	case BucketReplicationSourceNotVersioned:
 		apiErr = ErrReplicationSourceNotVersionedError
 	case BucketQuotaExceeded:

cmd/api-headers.go

@@ -47,7 +47,9 @@ func setEventStreamHeaders(w http.ResponseWriter) {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
+	// Set the "Server" http header.
+	w.Header().Set(xhttp.ServerInfo, "MinIO")
+
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerRegion; region != "" {
@@ -115,10 +117,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set tag count if object has tags
-	tags, _ := url.ParseQuery(objInfo.UserTags)
+	if len(objInfo.UserTags) > 0 {
-	tagCount := len(tags)
+		tags, _ := url.ParseQuery(objInfo.UserTags)
-	if tagCount > 0 {
+		if len(tags) > 0 {
-		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		}
 	}
 
 	// Set all other user defined metadata.
@@ -154,19 +157,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	if opts.PartNumber > 0 {
-		var start, end int64
+		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
-		for i := 0; i < len(objInfo.Parts) && i < opts.PartNumber; i++ {
+	}
-			start = end
+
-			end = start + objInfo.Parts[i].ActualSize - 1
+	// For providing ranged content
-		}
+	start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
-		rs = &HTTPRangeSpec{Start: start, End: end}
+	if err != nil {
-		rangeLen = end - start + 1
+		return err
-	} else {
-		// for providing ranged content
-		start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
-		if err != nil {
-			return err
-		}
 	}
 
 	// Set content length.
@@ -197,6 +194,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 				fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),
 			}
 		}
+		if objInfo.TransitionStatus == lifecycle.TransitionComplete {
+			w.Header()[xhttp.AmzStorageClass] = []string{objInfo.StorageClass}
+		}
 	}
 
 	return nil

cmd/api-response.go

@@ -35,11 +35,11 @@ import (
 
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z"                     // Reply date format with nanosecond precision.
-	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxObjectList     = metacacheBlockSize - (metacacheBlockSize / 10) // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
+	maxDeleteList     = 10000                                          // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
+	maxUploadsList    = 10000                                          // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
+	maxPartsList      = 10000                                          // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -388,7 +388,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 	}
 	proto := handlers.GetSourceScheme(r)
 	if proto == "" {
-		proto = getURLScheme(globalIsSSL)
+		proto = getURLScheme(globalIsTLS)
 	}
 	u := &url.URL{
 		Host:   r.Host,
@@ -703,10 +703,6 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, err
 }
 
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	if newObjectLayerFn() == nil {
-		// Server still in safe mode.
-		w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-	}
 	setCommonHeaders(w)
 	if mType != mimeNone {
 		w.Header().Set(xhttp.ContentType, string(mType))
@@ -773,10 +769,6 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalBrowserEnabled {
-			if newObjectLayerFn() == nil {
-				// server still in safe mode.
-				w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-			}
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return

cmd/api-router.go

@@ -32,31 +32,24 @@ func newHTTPServerFn() *xhttp.Server {
 	return globalHTTPServer
 }
 
-func newObjectLayerWithoutSafeModeFn() ObjectLayer {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
-	return globalObjectAPI
-}
-
 func newObjectLayerFn() ObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-	if globalSafeMode {
-		return nil
-	}
 	return globalObjectAPI
 }
 
 func newCachedObjectLayerFn() CacheObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-
-	if globalSafeMode {
-		return nil
-	}
 	return globalCacheObjectAPI
 }
 
+func setObjectLayer(o ObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = o
+	globalObjLayerMutex.Unlock()
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
@@ -314,6 +307,9 @@ func registerAPIRouter(router *mux.Router) {
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(
 			maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler))))
+		// PostRestoreObject
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
+			maxClients(collectAPIStats("restoreobject", httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
 	}
 
 	/// Root operation
@@ -333,7 +329,7 @@ func registerAPIRouter(router *mux.Router) {
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
-	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler))
+	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
 
 }
 

cmd/auth-handler.go

@@ -151,9 +151,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	return cred, claims, owner, ErrNone
 }
 
-// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
+// checkAdminRequestAuth checks for authentication and authorization for the incoming
-// It does not accept presigned or JWT or anonymous requests.
+// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
-func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+// are automatically rejected.
+func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
 	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
@@ -333,8 +334,12 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Populate payload again to handle it in HTTP handler.
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
 
-	if cred.AccessKey == "" {
+	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
+		// Anonymous checks are not meant for ListBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Action:          action,
@@ -378,12 +383,13 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Request is allowed return the appropriate access key.
 		return cred.AccessKey, owner, ErrNone
 	}
+
 	if action == policy.ListBucketVersionsAction {
 		// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
 		// verify as a fallback.
 		if globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          iampolicy.Action(policy.ListBucketAction),
+			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			ObjectName:      objectName,
@@ -554,7 +560,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		if retMode == objectlock.RetGovernance && byPassSet {
 			byPassSet = globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
-				Action:          policy.Action(policy.BypassGovernanceRetentionAction),
+				Action:          policy.BypassGovernanceRetentionAction,
 				BucketName:      bucketName,
 				ConditionValues: conditions,
 				IsOwner:         false,
@@ -563,7 +569,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		}
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.Action(policy.PutObjectRetentionAction),
+			Action:          policy.PutObjectRetentionAction,
 			BucketName:      bucketName,
 			ConditionValues: conditions,
 			IsOwner:         false,
@@ -586,7 +592,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	if retMode == objectlock.RetGovernance && byPassSet {
 		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.BypassGovernanceRetentionAction,
+			Action:          iampolicy.BypassGovernanceRetentionAction,
 			BucketName:      bucketName,
 			ObjectName:      objectName,
 			ConditionValues: conditions,
@@ -596,7 +602,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
-		Action:          policy.PutObjectRetentionAction,
+		Action:          iampolicy.PutObjectRetentionAction,
 		BucketName:      bucketName,
 		ConditionValues: conditions,
 		ObjectName:      objectName,
@@ -614,7 +620,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
+func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	switch atype {
@@ -635,6 +641,10 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 		return s3Err
 	}
 
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
+
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.
 	// Can happen when bucket has default lock config set

cmd/auth-handler_test.go

@@ -421,7 +421,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuthType(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/background-heal-ops.go

@@ -18,9 +18,9 @@ package cmd
 
 import (
 	"context"
-	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -54,20 +54,33 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
-func waitForLowHTTPReq(tolerance int32, maxWait time.Duration) {
+func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
-	const wait = 10 * time.Millisecond
+	// No need to wait run at full speed.
-	waitCount := maxWait / wait
+	if maxIO <= 0 {
+		return
+	}
+
+	// At max 10 attempts to wait with 100 millisecond interval before proceeding
+	waitCount := 10
+	waitTick := 100 * time.Millisecond
 
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	tolerance += int32(globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers())
+	maxIOFn := func() int {
+		return maxIO + globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers()
+	}
 
 	if httpServer := newHTTPServerFn(); httpServer != nil {
 		// Any requests in progress, delay the heal.
-		for (httpServer.GetRequestCount() >= tolerance) &&
+		for httpServer.GetRequestCount() >= maxIOFn() {
-			waitCount > 0 {
+			time.Sleep(waitTick)
 			waitCount--
-			time.Sleep(wait)
+			if waitCount == 0 {
+				if intDataUpdateTracker.debug {
+					logger.Info("waitForLowHTTPReq: waited %d times, resuming", waitCount)
+				}
+				break
+			}
 		}
 	}
 }
@@ -81,9 +94,6 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 				break
 			}
 
-			// Wait and proceed if there are active requests
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
-
 			var res madmin.HealResultItem
 			var err error
 			switch {
@@ -92,13 +102,10 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			case task.bucket == SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
 			case task.bucket != "" && task.object == "":
-				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts.DryRun, task.opts.Remove)
+				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
 			case task.bucket != "" && task.object != "":
 				res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
 			}
-			if task.bucket != "" && task.object != "" {
-				ObjectPathUpdated(path.Join(task.bucket, task.object))
-			}
 			task.responseCh <- healResult{result: res, err: err}
 
 		case <-h.doneCh:

cmd/background-newdisks-heal-ops.go

@@ -20,10 +20,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/console"
 )
 
 const (
@@ -39,23 +41,14 @@ type healingTracker struct {
 }
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	z, ok := objAPI.(*erasureZones)
+	z, ok := objAPI.(*erasureServerPools)
 	if !ok {
 		return
 	}
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	var bgSeq *healSequence
+	bgSeq := mustGetHealSequence(ctx)
-	var found bool
-
-	for {
-		bgSeq, found = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-		if found {
-			break
-		}
-		time.Sleep(time.Second)
-	}
 
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
@@ -64,7 +57,8 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 			drivesToHeal, defaultMonitorNewDiskInterval))
 
 		// Heal any disk format and metadata early, if possible.
-		if err := bgSeq.healDiskMeta(); err != nil {
+		// Start with format healing
+		if err := bgSeq.healDiskFormat(); err != nil {
 			if newObjectLayerFn() != nil {
 				// log only in situations, when object layer
 				// has fully initialized.
@@ -73,6 +67,14 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
+	if err := bgSeq.healDiskMeta(objAPI); err != nil {
+		if newObjectLayerFn() != nil {
+			// log only in situations, when object layer
+			// has fully initialized.
+			logger.LogIf(bgSeq.ctx, err)
+		}
+	}
+
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 
@@ -101,20 +103,24 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealRoutine = newHealRoutine()
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
 }
 
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
+	diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval)
+	defer diskCheckTimer.Stop()
+wait:
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.After(defaultMonitorNewDiskInterval):
+		case <-diskCheckTimer.C:
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
+			// Reset to next interval.
+			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 
 			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
 
@@ -129,12 +135,16 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.zones))
+				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
-				for i := range z.zones {
+				for i := range z.serverPools {
 					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
 
+			if serverDebugLog {
+				console.Debugf("disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+			}
+
 			// heal only if new disks found.
 			for _, endpoint := range healDisks {
 				disk, format, err := connectEndpoint(endpoint)
@@ -149,7 +159,10 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				}
 
 				// Calculate the set index where the current endpoint belongs
-				setIndex, _, err := findDiskIndex(z.zones[zoneIdx].format, format)
+				z.serverPools[zoneIdx].erasureDisksMu.RLock()
+				// Protect reading reference format.
+				setIndex, _, err := findDiskIndex(z.serverPools[zoneIdx].format, format)
+				z.serverPools[zoneIdx].erasureDisksMu.RUnlock()
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					continue
@@ -158,13 +171,39 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
 			}
 
-			buckets, _ := z.ListBucketsHeal(ctx)
+			buckets, _ := z.ListBuckets(ctx)
+
+			// Heal latest buckets first.
+			sort.Slice(buckets, func(i, j int) bool {
+				return buckets[i].Created.After(buckets[j].Created)
+			})
+
 			for i, setMap := range erasureSetInZoneDisksToHeal {
 				for setIndex, disks := range setMap {
 					for _, disk := range disks {
 						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
 
-						lbDisks := z.zones[i].sets[setIndex].getLoadBalancedNDisks(z.zones[i].listTolerancePerSet)
+						// So someone changed the drives underneath, healing tracker missing.
+						if !disk.Healing() {
+							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s zone", disk, humanize.Ordinal(i+1))
+							diskID, err := disk.GetDiskID()
+							if err != nil {
+								logger.LogIf(ctx, err)
+								// reading format.json failed or not found, proceed to look
+								// for new disks to be healed again, we cannot proceed further.
+								goto wait
+							}
+
+							if err := saveHealingTracker(disk, diskID); err != nil {
+								logger.LogIf(ctx, err)
+								// Unable to write healing tracker, permission denied or some
+								// other unexpected error occurred. Proceed to look for new
+								// disks to be healed again, we cannot proceed further.
+								goto wait
+							}
+						}
+
+						lbDisks := z.serverPools[i].sets[setIndex].getOnlineDisks()
 						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
 							logger.LogIf(ctx, err)
 							continue
@@ -172,8 +211,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 
 						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
 
-						if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
+						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
-							healingTrackerFilename); err != nil {
+							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
 							logger.LogIf(ctx, err)
 							continue
 						}

cmd/bootstrap-peer-server.go

@@ -18,9 +18,7 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -54,7 +52,7 @@ type bootstrapRESTServer struct{}
 type ServerSystemConfig struct {
 	MinioPlatform  string
 	MinioRuntime   string
-	MinioEndpoints EndpointZones
+	MinioEndpoints EndpointServerPools
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -161,9 +159,9 @@ func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSyst
 	return srcCfg.Diff(recvCfg)
 }
 
-func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointZones)
+	clnts := newBootstrapRESTClients(endpointServerPools)
 	var onlineServers int
 	var offlineEndpoints []string
 	var retries int
@@ -198,10 +196,10 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
 	return nil
 }
 
-func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient {
+func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstrapRESTClient {
 	seenHosts := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
 			if seenHosts.Contains(endpoint.Host) {
 				continue
@@ -225,26 +223,8 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		Path:   bootstrapRESTPath,
 	}
 
-	var tlsConfig *tls.Config
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
-	if globalIsSSL {
+	restClient.HealthCheckFn = nil
-		tlsConfig = &tls.Config{
-			ServerName: endpoint.Hostname(),
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
-	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
-	restClient.HealthCheckFn = func() bool {
-		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
-		// Instantiate a new rest client for healthcheck
-		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
-		xhttp.DrainBody(respBody)
-		cancel()
-		var ne *rest.NetworkError
-		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
-	}
 
 	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/bucket-encryption.go

@@ -34,7 +34,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 // Get - gets bucket encryption config for the given bucket.
 func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-handlers.go

@@ -28,6 +28,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -37,6 +38,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/bucket/replication"
@@ -383,6 +385,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
+	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
+	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
+
 	// Before proceeding validate if bucket exists.
 	_, err := objectAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
@@ -400,7 +406,18 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if api.CacheAPI() != nil {
 		getObjectInfoFn = api.CacheAPI().GetObjectInfo
 	}
-
+	var (
+		hasLockEnabled, hasLifecycleConfig bool
+		goi                                ObjectInfo
+		gerr                               error
+	)
+	replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects)
+	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+		hasLockEnabled = true
+	}
+	if _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket); err == nil {
+		hasLifecycleConfig = true
+	}
 	dErrs := make([]DeleteError, len(deleteObjects.Objects))
 	for index, object := range deleteObjects.Objects {
 		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
@@ -417,10 +434,47 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 			continue
 		}
+		if object.VersionID != "" && object.VersionID != nullVersionID {
+			if _, err := uuid.Parse(object.VersionID); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("invalid version-id specified %w", err))
+				apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion)
+				dErrs[index] = DeleteError{
+					Code:      apiErr.Code,
+					Message:   apiErr.Description,
+					Key:       object.ObjectName,
+					VersionID: object.VersionID,
+				}
+				continue
+			}
+		}
 
+		if replicateDeletes || hasLockEnabled || hasLifecycleConfig {
+			goi, gerr = getObjectInfoFn(ctx, bucket, object.ObjectName, ObjectOptions{
+				VersionID: object.VersionID,
+			})
+		}
+		if hasLifecycleConfig && gerr == nil {
+			object.PurgeTransitioned = goi.TransitionStatus
+		}
+		if replicateDeletes {
+			delMarker, replicate := checkReplicateDelete(ctx, bucket, ObjectToDelete{
+				ObjectName: object.ObjectName,
+				VersionID:  object.VersionID,
+			}, goi, gerr)
+			if replicate {
+				if object.VersionID != "" {
+					object.VersionPurgeStatus = Pending
+					if delMarker {
+						object.DeleteMarkerVersionID = object.VersionID
+					}
+				} else {
+					object.DeleteMarkerReplicationStatus = string(replication.Pending)
+				}
+			}
+		}
 		if object.VersionID != "" {
-			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+			if hasLockEnabled {
-				if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, getObjectInfoFn); apiErrCode != ErrNone {
+				if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
 					apiErr := errorCodes.ToAPIErr(apiErrCode)
 					dErrs[index] = DeleteError{
 						Code:      apiErr.Code,
@@ -454,15 +508,25 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		Versioned:        globalBucketVersioningSys.Enabled(bucket),
 		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
-
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
-		dindex := objectsToDelete[deleteList[i]]
+		dindex := objectsToDelete[ObjectToDelete{
-		apiErr := toAPIError(ctx, errs[i])
+			ObjectName:                    dObjects[i].ObjectName,
-		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
+			VersionID:                     dObjects[i].VersionID,
+			DeleteMarkerVersionID:         dObjects[i].DeleteMarkerVersionID,
+			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus,
+			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
+			PurgeTransitioned:             dObjects[i].PurgeTransitioned,
+		}]
+		if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
+			if replicateDeletes {
+				dObjects[i].DeleteMarkerReplicationStatus = deleteList[i].DeleteMarkerReplicationStatus
+				dObjects[i].VersionPurgeStatus = deleteList[i].VersionPurgeStatus
+			}
 			deletedObjects[dindex] = dObjects[i]
 			continue
 		}
+		apiErr := toAPIError(ctx, errs[i])
 		dErrs[dindex] = DeleteError{
 			Code:      apiErr.Code,
 			Message:   apiErr.Description,
@@ -484,22 +548,46 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
+	for _, dobj := range deletedObjects {
+		if replicateDeletes {
+			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
+				globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+					DeletedObject: dobj,
+					Bucket:        bucket,
+				})
+			}
+		}
 
+		if hasLifecycleConfig && dobj.PurgeTransitioned == lifecycle.TransitionComplete { // clean up transitioned tier
+			action := lifecycle.DeleteAction
+			if dobj.VersionID != "" {
+				action = lifecycle.DeleteVersionAction
+			}
+			deleteTransitionedObject(ctx, newObjectLayerFn(), bucket, dobj.ObjectName, lifecycle.ObjectOpts{
+				Name:         dobj.ObjectName,
+				VersionID:    dobj.VersionID,
+				DeleteMarker: dobj.DeleteMarker,
+			}, action, true)
+		}
+
+	}
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
+		eventName := event.ObjectRemovedDelete
+
 		objInfo := ObjectInfo{
 			Name:      dobj.ObjectName,
 			VersionID: dobj.VersionID,
 		}
+
 		if dobj.DeleteMarker {
-			objInfo = ObjectInfo{
+			objInfo.DeleteMarker = dobj.DeleteMarker
-				Name:         dobj.ObjectName,
+			objInfo.VersionID = dobj.DeleteMarkerVersionID
-				DeleteMarker: dobj.DeleteMarker,
+			eventName = event.ObjectRemovedDeleteMarkerCreated
-				VersionID:    dobj.DeleteMarkerVersionID,
-			}
 		}
+
 		sendEvent(eventArgs{
-			EventName:    event.ObjectRemovedDelete,
+			EventName:    eventName,
 			BucketName:   bucket,
 			Object:       objInfo,
 			ReqParams:    extractReqParams(r),
@@ -660,7 +748,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
+	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -807,7 +895,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
 		// This request header needs to be set prior to setting ObjectOptions
 		if !crypto.SSEC.IsRequested(r.Header) {
-			r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+			r.Header.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 		}
 	}
 
@@ -819,7 +907,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsRequested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
 				return
@@ -1289,7 +1377,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
 	if err = globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-handlers_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -837,7 +836,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
-			fmt.Println(string(testCase.expectedContent), string(actualContent))
+			t.Log(string(testCase.expectedContent), string(actualContent))
 			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
 		}
 	}

cmd/bucket-lifecycle-handlers.go

@@ -78,6 +78,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Validate the transition storage ARNs
+	if err = validateLifecycleTransition(ctx, bucket, bucketLifecycle); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	configData, err := xml.Marshal(bucketLifecycle)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/bucket-lifecycle.go

@@ -17,7 +17,25 @@
 package cmd
 
 import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"strings"
+	"time"
+
+	miniogo "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	sse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/s3select"
 )
 
 const (
@@ -31,7 +49,7 @@ type LifecycleSys struct{}
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -46,3 +64,591 @@ func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err er
 func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
+
+type transitionState struct {
+	// add future metrics here
+	transitionCh chan ObjectInfo
+}
+
+func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
+	select {
+	case t.transitionCh <- oi:
+	default:
+	}
+}
+
+var (
+	globalTransitionState      *transitionState
+	globalTransitionConcurrent = runtime.GOMAXPROCS(0) / 2
+)
+
+func newTransitionState() *transitionState {
+
+	// fix minimum concurrent transition to 1 for single CPU setup
+	if globalTransitionConcurrent == 0 {
+		globalTransitionConcurrent = 1
+	}
+	ts := &transitionState{
+		transitionCh: make(chan ObjectInfo, 10000),
+	}
+	go func() {
+		<-GlobalContext.Done()
+		close(ts.transitionCh)
+	}()
+	return ts
+}
+
+// addWorker creates a new worker to process tasks
+func (t *transitionState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
+	// Add a new worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case oi, ok := <-t.transitionCh:
+				if !ok {
+					return
+				}
+				if err := transitionObject(ctx, objectAPI, oi); err != nil {
+					logger.LogIf(ctx, err)
+				}
+			}
+		}
+	}()
+}
+
+func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
+	if globalTransitionState == nil {
+		return
+	}
+
+	// Start with globalTransitionConcurrent.
+	for i := 0; i < globalTransitionConcurrent; i++ {
+		globalTransitionState.addWorker(ctx, objectAPI)
+	}
+}
+
+func validateLifecycleTransition(ctx context.Context, bucket string, lfc *lifecycle.Lifecycle) error {
+	for _, rule := range lfc.Rules {
+		if rule.Transition.StorageClass != "" {
+			sameTarget, destbucket, err := validateTransitionDestination(ctx, bucket, rule.Transition.StorageClass)
+			if err != nil {
+				return err
+			}
+			if sameTarget && destbucket == bucket {
+				return fmt.Errorf("Transition destination cannot be the same as the source bucket")
+			}
+		}
+	}
+	return nil
+}
+
+// validateTransitionDestination returns error if transition destination bucket missing or not configured
+// It also returns true if transition destination is same as this server.
+func validateTransitionDestination(ctx context.Context, bucket string, targetLabel string) (bool, string, error) {
+	tgt := globalBucketTargetSys.GetRemoteTargetWithLabel(ctx, bucket, targetLabel)
+	if tgt == nil {
+		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	arn, err := madmin.ParseARN(tgt.Arn)
+	if err != nil {
+		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	if arn.Type != madmin.ILMService {
+		return false, "", BucketRemoteArnTypeInvalid{}
+	}
+	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, tgt.Arn)
+	if clnt == nil {
+		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	if found, _ := clnt.BucketExists(ctx, arn.Bucket); !found {
+		return false, "", BucketRemoteDestinationNotFound{Bucket: arn.Bucket}
+	}
+	sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+	return sameTarget, arn.Bucket, nil
+}
+
+// transitionSC returns storage class label for this bucket
+func transitionSC(ctx context.Context, bucket string) string {
+	cfg, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+	if err != nil {
+		return ""
+	}
+	for _, rule := range cfg.Rules {
+		if rule.Status == Disabled {
+			continue
+		}
+		if rule.Transition.StorageClass != "" {
+			return rule.Transition.StorageClass
+		}
+	}
+	return ""
+}
+
+// return true if ARN representing transition storage class is present in a active rule
+// for the lifecycle configured on this bucket
+func transitionSCInUse(ctx context.Context, lfc *lifecycle.Lifecycle, bucket, arnStr string) bool {
+	tgtLabel := globalBucketTargetSys.GetRemoteLabelWithArn(ctx, bucket, arnStr)
+	if tgtLabel == "" {
+		return false
+	}
+	for _, rule := range lfc.Rules {
+		if rule.Status == Disabled {
+			continue
+		}
+		if rule.Transition.StorageClass != "" && rule.Transition.StorageClass == tgtLabel {
+			return true
+		}
+	}
+	return false
+}
+
+// set PutObjectOptions for PUT operation to transition data to target cluster
+func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+	meta := make(map[string]string)
+
+	tag, err := tags.ParseObjectTags(objInfo.UserTags)
+	if err != nil {
+		return
+	}
+	putOpts = miniogo.PutObjectOptions{
+		UserMetadata:    meta,
+		UserTags:        tag.ToMap(),
+		ContentType:     objInfo.ContentType,
+		ContentEncoding: objInfo.ContentEncoding,
+		StorageClass:    objInfo.StorageClass,
+		Internal: miniogo.AdvancedPutOptions{
+			SourceVersionID: objInfo.VersionID,
+			SourceMTime:     objInfo.ModTime,
+			SourceETag:      objInfo.ETag,
+		},
+	}
+	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+		rmode := miniogo.RetentionMode(mode)
+		putOpts.Mode = rmode
+	}
+	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
+		rdate, err := time.Parse(time.RFC3339, retainDateStr)
+		if err != nil {
+			return
+		}
+		putOpts.RetainUntilDate = rdate
+	}
+	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
+	}
+
+	return
+}
+
+// handle deletes of transitioned objects or object versions when one of the following is true:
+// 1. temporarily restored copies of objects (restored with the PostRestoreObject API) expired.
+// 2. life cycle expiry date is met on the object.
+// 3. Object is removed through DELETE api call
+func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, action lifecycle.Action, isDeleteTierOnly bool) error {
+	if lcOpts.TransitionStatus == "" && !isDeleteTierOnly {
+		return nil
+	}
+	lc, err := globalLifecycleSys.Get(bucket)
+	if err != nil {
+		return err
+	}
+	arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lcOpts)
+	if arn == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
+	if tgt == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+
+	var opts ObjectOptions
+	opts.Versioned = globalBucketVersioningSys.Enabled(bucket)
+	opts.VersionID = lcOpts.VersionID
+	switch action {
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		// delete locally restored copy of object or object version
+		// from the source, while leaving metadata behind. The data on
+		// transitioned tier lies untouched and still accessible
+		opts.TransitionStatus = lcOpts.TransitionStatus
+		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
+		return err
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+		// When an object is past expiry, delete the data from transitioned tier and
+		// metadata from source
+		if err := tgt.RemoveObject(context.Background(), arn.Bucket, object, miniogo.RemoveObjectOptions{VersionID: lcOpts.VersionID}); err != nil {
+			logger.LogIf(ctx, err)
+		}
+
+		if isDeleteTierOnly {
+			return nil
+		}
+		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
+		if err != nil {
+			return err
+		}
+		eventName := event.ObjectRemovedDelete
+		if lcOpts.DeleteMarker {
+			eventName = event.ObjectRemovedDeleteMarkerCreated
+		}
+		objInfo := ObjectInfo{
+			Name:         object,
+			VersionID:    lcOpts.VersionID,
+			DeleteMarker: lcOpts.DeleteMarker,
+		}
+		// Notify object deleted event.
+		sendEvent(eventArgs{
+			EventName:  eventName,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [ILM-EXPIRY]",
+		})
+	}
+
+	// should never reach here
+	return nil
+}
+
+// transition object to target specified by the transition ARN. When an object is transitioned to another
+// storage specified by the transition ARN, the metadata is left behind on source cluster and original content
+// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
+// to the transition tier without decrypting or re-encrypting.
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo ObjectInfo) error {
+	lc, err := globalLifecycleSys.Get(objInfo.Bucket)
+	if err != nil {
+		return err
+	}
+	lcOpts := lifecycle.ObjectOpts{
+		Name:     objInfo.Name,
+		UserTags: objInfo.UserTags,
+	}
+	arn := getLifecycleTransitionTargetArn(ctx, lc, objInfo.Bucket, lcOpts)
+	if arn == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
+	if tgt == nil {
+		return fmt.Errorf("remote target not configured")
+	}
+
+	gr, err := objectAPI.GetObjectNInfo(ctx, objInfo.Bucket, objInfo.Name, nil, http.Header{}, readLock, ObjectOptions{
+		VersionID:        objInfo.VersionID,
+		TransitionStatus: lifecycle.TransitionPending,
+	})
+	if err != nil {
+		return err
+	}
+	oi := gr.ObjInfo
+
+	if oi.TransitionStatus == lifecycle.TransitionComplete {
+		return nil
+	}
+
+	putOpts := putTransitionOpts(oi)
+	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, "", "", putOpts); err != nil {
+		return err
+	}
+	gr.Close()
+
+	var opts ObjectOptions
+	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
+	opts.VersionID = oi.VersionID
+	opts.TransitionStatus = lifecycle.TransitionComplete
+	eventName := event.ObjectTransitionComplete
+
+	_, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	if err != nil {
+		eventName = event.ObjectTransitionFailed
+	}
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: oi.Bucket,
+		Object: ObjectInfo{
+			Name:      oi.Name,
+			VersionID: opts.VersionID,
+		},
+		Host: "Internal: [ILM-Transition]",
+	})
+	return err
+}
+
+// getLifecycleTransitionTargetArn returns transition ARN for storage class specified in the config.
+func getLifecycleTransitionTargetArn(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) *madmin.ARN {
+	for _, rule := range lc.FilterActionableRules(obj) {
+		if rule.Transition.StorageClass != "" {
+			return globalBucketTargetSys.GetRemoteArnWithLabel(ctx, bucket, rule.Transition.StorageClass)
+		}
+	}
+	return nil
+}
+
+// getTransitionedObjectReader returns a reader from the transitioned tier.
+func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	var lc *lifecycle.Lifecycle
+	lc, err = globalLifecycleSys.Get(bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lifecycle.ObjectOpts{
+		Name:         object,
+		UserTags:     oi.UserTags,
+		ModTime:      oi.ModTime,
+		VersionID:    oi.VersionID,
+		DeleteMarker: oi.DeleteMarker,
+		IsLatest:     oi.IsLatest,
+	})
+	if arn == nil {
+		return nil, fmt.Errorf("remote target not configured")
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
+	if tgt == nil {
+		return nil, fmt.Errorf("remote target not configured")
+	}
+	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	if err != nil {
+		return nil, ErrorRespToObjectError(err, bucket, object)
+	}
+	gopts := miniogo.GetObjectOptions{VersionID: opts.VersionID}
+
+	// get correct offsets for encrypted object
+	if off >= 0 && length >= 0 {
+		if err := gopts.SetRange(off, off+length-1); err != nil {
+			return nil, ErrorRespToObjectError(err, bucket, object)
+		}
+	}
+
+	reader, _, _, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
+	if err != nil {
+		return nil, err
+	}
+	closeReader := func() { reader.Close() }
+
+	return fn(reader, h, opts.CheckPrecondFn, closeReader)
+}
+
+// RestoreRequestType represents type of restore.
+type RestoreRequestType string
+
+const (
+	// SelectRestoreRequest specifies select request. This is the only valid value
+	SelectRestoreRequest RestoreRequestType = "SELECT"
+)
+
+// Encryption specifies encryption setting on restored bucket
+type Encryption struct {
+	EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"`
+	KMSContext     string           `xml:"KMSContext,omitempty"`
+	KMSKeyID       string           `xml:"KMSKeyId,omitempty"`
+}
+
+// MetadataEntry denotes name and value.
+type MetadataEntry struct {
+	Name  string `xml:"Name"`
+	Value string `xml:"Value"`
+}
+
+// S3Location specifies s3 location that receives result of a restore object request
+type S3Location struct {
+	BucketName   string          `xml:"BucketName,omitempty"`
+	Encryption   Encryption      `xml:"Encryption,omitempty"`
+	Prefix       string          `xml:"Prefix,omitempty"`
+	StorageClass string          `xml:"StorageClass,omitempty"`
+	Tagging      *tags.Tags      `xml:"Tagging,omitempty"`
+	UserMetadata []MetadataEntry `xml:"UserMetadata"`
+}
+
+// OutputLocation specifies bucket where object needs to be restored
+type OutputLocation struct {
+	S3 S3Location `xml:"S3,omitempty"`
+}
+
+// IsEmpty returns true if output location not specified.
+func (o *OutputLocation) IsEmpty() bool {
+	return o.S3.BucketName == ""
+}
+
+// SelectParameters specifies sql select parameters
+type SelectParameters struct {
+	s3select.S3Select
+}
+
+// IsEmpty returns true if no select parameters set
+func (sp *SelectParameters) IsEmpty() bool {
+	return sp == nil || sp.S3Select == s3select.S3Select{}
+}
+
+var (
+	selectParamsXMLName = "SelectParameters"
+)
+
+// UnmarshalXML - decodes XML data.
+func (sp *SelectParameters) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Essentially the same as S3Select barring the xml name.
+	if start.Name.Local == selectParamsXMLName {
+		start.Name = xml.Name{Space: "", Local: "SelectRequest"}
+	}
+	return sp.S3Select.UnmarshalXML(d, start)
+}
+
+// RestoreObjectRequest - xml to restore a transitioned object
+type RestoreObjectRequest struct {
+	XMLName          xml.Name           `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RestoreRequest" json:"-"`
+	Days             int                `xml:"Days,omitempty"`
+	Type             RestoreRequestType `xml:"Type,omitempty"`
+	Tier             string             `xml:"Tier,-"`
+	Description      string             `xml:"Description,omitempty"`
+	SelectParameters *SelectParameters  `xml:"SelectParameters,omitempty"`
+	OutputLocation   OutputLocation     `xml:"OutputLocation,omitempty"`
+}
+
+// Maximum 2MiB size per restore object request.
+const maxRestoreObjectRequestSize = 2 << 20
+
+// parseRestoreRequest parses RestoreObjectRequest from xml
+func parseRestoreRequest(reader io.Reader) (*RestoreObjectRequest, error) {
+	req := RestoreObjectRequest{}
+	if err := xml.NewDecoder(io.LimitReader(reader, maxRestoreObjectRequestSize)).Decode(&req); err != nil {
+		return nil, err
+	}
+	return &req, nil
+}
+
+// validate a RestoreObjectRequest as per AWS S3 spec https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html
+func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer) error {
+	if r.Type != SelectRestoreRequest && !r.SelectParameters.IsEmpty() {
+		return fmt.Errorf("Select parameters can only be specified with SELECT request type")
+	}
+	if r.Type == SelectRestoreRequest && r.SelectParameters.IsEmpty() {
+		return fmt.Errorf("SELECT restore request requires select parameters to be specified")
+	}
+
+	if r.Type != SelectRestoreRequest && !r.OutputLocation.IsEmpty() {
+		return fmt.Errorf("OutputLocation required only for SELECT request type")
+	}
+	if r.Type == SelectRestoreRequest && r.OutputLocation.IsEmpty() {
+		return fmt.Errorf("OutputLocation required for SELECT requests")
+	}
+
+	if r.Days != 0 && r.Type == SelectRestoreRequest {
+		return fmt.Errorf("Days cannot be specified with SELECT restore request")
+	}
+	if r.Days == 0 && r.Type != SelectRestoreRequest {
+		return fmt.Errorf("restoration days should be at least 1")
+	}
+	// Check if bucket exists.
+	if !r.OutputLocation.IsEmpty() {
+		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
+			return err
+		}
+		if r.OutputLocation.S3.Prefix == "" {
+			return fmt.Errorf("Prefix is a required parameter in OutputLocation")
+		}
+		if r.OutputLocation.S3.Encryption.EncryptionType != xhttp.AmzEncryptionAES {
+			return NotImplemented{}
+		}
+	}
+	return nil
+}
+
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data
+func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+	meta := make(map[string]string)
+	sc := rreq.OutputLocation.S3.StorageClass
+	if sc == "" {
+		sc = objInfo.StorageClass
+	}
+	meta[strings.ToLower(xhttp.AmzStorageClass)] = sc
+
+	if rreq.Type == SelectRestoreRequest {
+		for _, v := range rreq.OutputLocation.S3.UserMetadata {
+			if !strings.HasPrefix("x-amz-meta", strings.ToLower(v.Name)) {
+				meta["x-amz-meta-"+v.Name] = v.Value
+				continue
+			}
+			meta[v.Name] = v.Value
+		}
+		meta[xhttp.AmzObjectTagging] = rreq.OutputLocation.S3.Tagging.String()
+		if rreq.OutputLocation.S3.Encryption.EncryptionType != "" {
+			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+		}
+		return ObjectOptions{
+			Versioned:        globalBucketVersioningSys.Enabled(bucket),
+			VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+			UserDefined:      meta,
+		}
+	}
+	for k, v := range objInfo.UserDefined {
+		meta[k] = v
+	}
+	meta[xhttp.AmzObjectTagging] = objInfo.UserTags
+
+	return ObjectOptions{
+		Versioned:        globalBucketVersioningSys.Enabled(bucket),
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+		UserDefined:      meta,
+		VersionID:        objInfo.VersionID,
+		MTime:            objInfo.ModTime,
+		Expires:          objInfo.Expires,
+	}
+}
+
+var (
+	errRestoreHDRMissing   = fmt.Errorf("x-amz-restore header not found")
+	errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
+)
+
+// parse x-amz-restore header from user metadata to get the status of ongoing request and expiry of restoration
+// if any. This header value is of format: ongoing-request=true|false, expires=time
+func parseRestoreHeaderFromMeta(meta map[string]string) (ongoing bool, expiry time.Time, err error) {
+	restoreHdr, ok := meta[xhttp.AmzRestore]
+	if !ok {
+		return ongoing, expiry, errRestoreHDRMissing
+	}
+	rslc := strings.SplitN(restoreHdr, ",", 2)
+	if len(rslc) != 2 {
+		return ongoing, expiry, errRestoreHDRMalformed
+	}
+	rstatusSlc := strings.SplitN(rslc[0], "=", 2)
+	if len(rstatusSlc) != 2 {
+		return ongoing, expiry, errRestoreHDRMalformed
+	}
+	rExpSlc := strings.SplitN(rslc[1], "=", 2)
+	if len(rExpSlc) != 2 {
+		return ongoing, expiry, errRestoreHDRMalformed
+	}
+
+	expiry, err = time.Parse(http.TimeFormat, rExpSlc[1])
+	if err != nil {
+		return
+	}
+	return rstatusSlc[1] == "true", expiry, nil
+}
+
+// restoreTransitionedObject is similar to PostObjectRestore from AWS GLACIER
+// storage class. When PostObjectRestore API is called, a temporary copy of the object
+// is restored locally to the bucket on source cluster until the restore expiry date.
+// The copy that was transitioned continues to reside in the transitioned tier.
+func restoreTransitionedObject(ctx context.Context, bucket, object string, objAPI ObjectLayer, objInfo ObjectInfo, rreq *RestoreObjectRequest, restoreExpiry time.Time) error {
+	var rs *HTTPRangeSpec
+	gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, objInfo, ObjectOptions{
+		VersionID: objInfo.VersionID})
+	if err != nil {
+		return err
+	}
+	defer gr.Close()
+	hashReader, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size, globalCLIContext.StrictS3Compat)
+	if err != nil {
+		return err
+	}
+	pReader := NewPutObjReader(hashReader, nil, nil)
+	opts := putRestoreOpts(bucket, object, rreq, objInfo)
+	opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
+	if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/bucket-listobjects-handlers.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -114,15 +113,6 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Forward the request using Source IP or bucket
-	forwardStr := handlers.GetSourceIPFromHeaders(r)
-	if forwardStr == "" {
-		forwardStr = bucket
-	}
-	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
-		return
-	}
-
 	listObjectVersions := objectAPI.ListObjectVersions
 
 	// Inititate a list object versions operation based on the input params.
@@ -145,7 +135,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
-// of the objects in a bucket. You can use the request parame<ters as selection
+// of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
 // NOTE: It is recommended that this API to be used for application development.
@@ -185,13 +175,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -207,9 +190,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -262,13 +242,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -282,13 +255,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
-	// The next continuation token has id@node_index format to optimize paginated listing
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
@@ -296,18 +263,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func getLocalNodeIndex() int {
-	if len(globalProxyEndpoints) == 0 {
-		return -1
-	}
-	for i, ep := range globalProxyEndpoints {
-		if ep.IsLocal {
-			return i
-		}
-	}
-	return -1
-}
-
 func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1

cmd/bucket-metadata-sys.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -49,6 +50,7 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
 	}
 	sys.Lock()
 	delete(sys.metadataMap, bucket)
+	globalBucketMonitor.DeleteBucket(bucket)
 	sys.Unlock()
 }
 
@@ -72,7 +74,7 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
 func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -167,10 +169,10 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		}
 		meta.ReplicationConfigXML = configData
 	case bucketTargetsFile:
-		if !globalIsErasure && !globalIsDistErasure {
+		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{bucket: meta.Name, bucketTargetsFile: bucketTargetsFile})
-			return NotImplemented{}
+		if err != nil {
+			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
-		meta.BucketTargetsConfigJSON = configData
 	default:
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
@@ -193,7 +195,6 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 // This function should only be used with
 // - GetBucketInfo
 // - ListBuckets
-// - ListBucketsHeal (only in case of erasure coding mode)
 // For all other bucket specific metadata, use the relevant
 // calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
@@ -275,7 +276,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
 	if globalIsGateway && globalGatewayName == NASBackendGateway {
 		// Only needed in case of NAS gateway.
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -313,7 +314,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -373,10 +374,24 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	return meta.bucketTargetConfig, nil
 }
 
+// GetBucketTarget returns the target for the bucket and arn.
+func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
+	targets, err := sys.GetBucketTargetsConfig(bucket)
+	if err != nil {
+		return madmin.BucketTarget{}, err
+	}
+	for _, t := range targets.Targets {
+		if t.Arn == arn {
+			return t, nil
+		}
+	}
+	return madmin.BucketTarget{}, errConfigNotFound
+}
+
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
@@ -418,16 +433,20 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 	}
 
 	// Load bucket metadata sys in background
-	go logger.LogIf(ctx, sys.load(ctx, buckets, objAPI))
+	go sys.load(ctx, buckets, objAPI)
 	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	g := errgroup.WithNErrs(len(buckets))
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
+			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
+				// Ensure heal opts for bucket metadata be deep healed all the time.
+				ScanMode: madmin.HealDeepScan,
+			})
 			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
 			if err != nil {
 				return err
@@ -440,22 +459,20 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	}
 	for _, err := range g.Wait() {
 		if err != nil {
-			return err
+			logger.LogIf(ctx, err)
 		}
 	}
-	return nil
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	count := 100 // load 100 bucket metadata at a time.
 	for {
 		if len(buckets) < count {
-			return sys.concurrentLoad(ctx, buckets, objAPI)
+			sys.concurrentLoad(ctx, buckets, objAPI)
-		}
+			return
-		if err := sys.concurrentLoad(ctx, buckets[:count], objAPI); err != nil {
-			return err
 		}
+		sys.concurrentLoad(ctx, buckets[:count], objAPI)
 		buckets = buckets[count:]
 	}
 }

cmd/bucket-metadata.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/binary"
 	"encoding/json"
 	"encoding/xml"
@@ -28,6 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -37,6 +39,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/sio"
 )
 
 const (
@@ -61,31 +64,33 @@ var (
 // bucketMetadataFormat refers to the format.
 // bucketMetadataVersion can be used to track a rolling upgrade of a field.
 type BucketMetadata struct {
-	Name                    string
+	Name                        string
-	Created                 time.Time
+	Created                     time.Time
-	LockEnabled             bool // legacy not used anymore.
+	LockEnabled                 bool // legacy not used anymore.
-	PolicyConfigJSON        []byte
+	PolicyConfigJSON            []byte
-	NotificationConfigXML   []byte
+	NotificationConfigXML       []byte
-	LifecycleConfigXML      []byte
+	LifecycleConfigXML          []byte
-	ObjectLockConfigXML     []byte
+	ObjectLockConfigXML         []byte
-	VersioningConfigXML     []byte
+	VersioningConfigXML         []byte
-	EncryptionConfigXML     []byte
+	EncryptionConfigXML         []byte
-	TaggingConfigXML        []byte
+	TaggingConfigXML            []byte
-	QuotaConfigJSON         []byte
+	QuotaConfigJSON             []byte
-	ReplicationConfigXML    []byte
+	ReplicationConfigXML        []byte
-	BucketTargetsConfigJSON []byte
+	BucketTargetsConfigJSON     []byte
+	BucketTargetsConfigMetaJSON []byte
 
 	// Unexported fields. Must be updated atomically.
-	policyConfig       *policy.Policy
+	policyConfig           *policy.Policy
-	notificationConfig *event.Config
+	notificationConfig     *event.Config
-	lifecycleConfig    *lifecycle.Lifecycle
+	lifecycleConfig        *lifecycle.Lifecycle
-	objectLockConfig   *objectlock.Config
+	objectLockConfig       *objectlock.Config
-	versioningConfig   *versioning.Versioning
+	versioningConfig       *versioning.Versioning
-	sseConfig          *bucketsse.BucketSSEConfig
+	sseConfig              *bucketsse.BucketSSEConfig
-	taggingConfig      *tags.Tags
+	taggingConfig          *tags.Tags
-	quotaConfig        *madmin.BucketQuota
+	quotaConfig            *madmin.BucketQuota
-	replicationConfig  *replication.Config
+	replicationConfig      *replication.Config
-	bucketTargetConfig *madmin.BucketTargets
+	bucketTargetConfig     *madmin.BucketTargets
+	bucketTargetConfigMeta map[string]string
 }
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
@@ -100,13 +105,18 @@ func newBucketMetadata(name string) BucketMetadata {
 		versioningConfig: &versioning.Versioning{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
-		bucketTargetConfig: &madmin.BucketTargets{},
+		bucketTargetConfig:     &madmin.BucketTargets{},
+		bucketTargetConfigMeta: make(map[string]string),
 	}
 }
 
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+	if name == "" {
+		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
+		return errors.New("bucket name cannot be empty")
+	}
 	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
@@ -128,22 +138,24 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 	}
 	// OK, parse data.
 	_, err = b.UnmarshalMsg(data[4:])
+	b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
 	return err
 }
 
 // loadBucketMetadata loads and migrates to bucket metadata.
 func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
-	err := b.Load(ctx, objectAPI, bucket)
+	err := b.Load(ctx, objectAPI, b.Name)
-	if err == nil {
+	if err != nil && !errors.Is(err, errConfigNotFound) {
-		return b, b.convertLegacyConfigs(ctx, objectAPI)
-	}
-
-	if err != errConfigNotFound {
 		return b, err
 	}
+
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
-	return b, b.convertLegacyConfigs(ctx, objectAPI)
+	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
+		return b, err
+	}
+	// migrate unencrypted remote targets
+	return b, b.migrateTargetConfig(ctx, objectAPI)
 }
 
 // parseAllConfigs will parse all configs and populate the private fields.
@@ -228,7 +240,8 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	}
 
 	if len(b.BucketTargetsConfigJSON) != 0 {
-		if err = json.Unmarshal(b.BucketTargetsConfigJSON, b.bucketTargetConfig); err != nil {
+		b.bucketTargetConfig, err = parseBucketTargetConfig(b.Name, b.BucketTargetsConfigJSON, b.BucketTargetsConfigMetaJSON)
+		if err != nil {
 			return err
 		}
 	} else {
@@ -348,6 +361,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 	if err != nil {
 		return err
 	}
+
 	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
 	return saveConfig(ctx, api, configFile, data)
 }
@@ -367,3 +381,76 @@ func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string)
 	}
 	return nil
 }
+
+// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
+func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
+	var err error
+	// early return if no targets or already encrypted
+	if len(b.BucketTargetsConfigJSON) == 0 || GlobalKMS == nil || len(b.BucketTargetsConfigMetaJSON) != 0 {
+		return nil
+	}
+
+	encBytes, metaBytes, err := encryptBucketMetadata(b.Name, b.BucketTargetsConfigJSON, crypto.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
+	if err != nil {
+		return err
+	}
+
+	b.BucketTargetsConfigJSON = encBytes
+	b.BucketTargetsConfigMetaJSON = metaBytes
+	return b.Save(ctx, objectAPI)
+}
+
+// encrypt bucket metadata if kms is configured.
+func encryptBucketMetadata(bucket string, input []byte, kmsContext crypto.Context) (output, metabytes []byte, err error) {
+	var sealedKey crypto.SealedKey
+	if GlobalKMS == nil {
+		output = input
+		return
+	}
+	var (
+		key    [32]byte
+		encKey []byte
+	)
+	metadata := make(map[string]string)
+	key, encKey, err = GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), kmsContext)
+	if err != nil {
+		return
+	}
+	outbuf := bytes.NewBuffer(nil)
+	objectKey := crypto.GenerateKey(key, rand.Reader)
+	sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
+	crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
+	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
+	if err != nil {
+		return output, metabytes, err
+	}
+	metabytes, err = json.Marshal(metadata)
+	if err != nil {
+		return
+	}
+	return outbuf.Bytes(), metabytes, nil
+}
+
+// decrypt bucket metadata if kms is configured.
+func decryptBucketMetadata(input []byte, bucket string, meta map[string]string, kmsContext crypto.Context) ([]byte, error) {
+	if GlobalKMS == nil {
+		return nil, errKMSNotConfigured
+	}
+	keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(meta)
+
+	if err != nil {
+		return nil, err
+	}
+	extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, kmsContext)
+	if err != nil {
+		return nil, err
+	}
+	var objectKey crypto.ObjectKey
+	if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, ""); err != nil {
+		return nil, err
+	}
+	outbuf := bytes.NewBuffer(nil)
+	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
+
+	return outbuf.Bytes(), err
+}

cmd/bucket-metadata_gen.go

@@ -102,6 +102,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
+		case "BucketTargetsConfigMetaJSON":
+			z.BucketTargetsConfigMetaJSON, err = dc.ReadBytes(z.BucketTargetsConfigMetaJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -115,9 +121,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 13
+	// map header, size 14
 	// write "Name"
-	err = en.Append(0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -246,15 +252,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 		return
 	}
+	// write "BucketTargetsConfigMetaJSON"
+	err = en.Append(0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.BucketTargetsConfigMetaJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 13
+	// map header, size 14
 	// string "Name"
-	o = append(o, 0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -292,6 +308,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "BucketTargetsConfigJSON"
 	o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
 	o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON)
+	// string "BucketTargetsConfigMetaJSON"
+	o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
 	return
 }
 
@@ -391,6 +410,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
+		case "BucketTargetsConfigMetaJSON":
+			z.BucketTargetsConfigMetaJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigMetaJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -405,6 +430,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON)
 	return
 }

cmd/bucket-object-lock.go

@@ -21,10 +21,12 @@ import (
 	"math"
 	"net/http"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/replication"
 )
 
 // BucketObjectLockSys - map of bucket and retention configuration.
@@ -33,7 +35,7 @@ type BucketObjectLockSys struct{}
 // Get - Get retention configuration.
 func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return r, errServerNotInitialized
 		}
@@ -80,24 +82,25 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
-func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
+func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) APIErrorCode {
 	opts, err := getOpts(ctx, r, bucket, object.ObjectName)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
 
 	opts.VersionID = object.VersionID
-
+	if gerr != nil { // error from GetObjectInfo
-	oi, err := getObjectInfoFn(ctx, bucket, object.ObjectName, opts)
+		switch gerr.(type) {
-	if err != nil {
-		switch err.(type) {
 		case MethodNotAllowed: // This happens usually for a delete marker
 			if oi.DeleteMarker {
 				// Delete marker should be present and valid.
 				return ErrNone
 			}
 		}
-		return toAPIErrorCode(ctx, err)
+		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) {
+			return ErrNone
+		}
+		return toAPIErrorCode(ctx, gerr)
 	}
 
 	lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
@@ -245,13 +248,13 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 // For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
 // For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
 // Both legal hold and retention can be applied independently on an object
-func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
+func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
 	var mode objectlock.RetMode
 	var retainDate objectlock.RetentionDate
 	var legalHold objectlock.ObjectLegalHold
 
-	retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header)
+	retentionRequested := objectlock.IsObjectLockRetentionRequested(rq.Header)
-	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header)
+	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(rq.Header)
 
 	retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
 	if err != nil {
@@ -267,25 +270,24 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		return mode, retainDate, legalHold, ErrNone
 	}
 
-	opts, err := getOpts(ctx, r, bucket, object)
+	opts, err := getOpts(ctx, rq, bucket, object)
 	if err != nil {
 		return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 	}
 
-	if opts.VersionID != "" {
+	replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
+
+	if opts.VersionID != "" && !replica {
 		if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
 			r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
-
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
 				logger.LogIf(ctx, err)
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
-
 			if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
-
 			mode = r.Mode
 			retainDate = r.RetainUntilDate
 			legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
@@ -298,17 +300,17 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 
 	if legalHoldRequested {
 		var lerr error
-		if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(r.Header); lerr != nil {
+		if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(rq.Header); lerr != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 	}
 
 	if retentionRequested {
-		legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(r.Header)
+		legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(rq.Header)
 		if err != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
-		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(r.Header)
+		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
 		if err != nil {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
@@ -317,7 +319,9 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		}
 		return rMode, rDate, legalHold, ErrNone
 	}
-
+	if replica { // replica inherits retention metadata only from source
+		return "", objectlock.RetentionDate{}, legalHold, ErrNone
+	}
 	if !retentionRequested && retentionCfg.Validity > 0 {
 		if retentionPermErr != ErrNone {
 			return mode, retainDate, legalHold, retentionPermErr

cmd/bucket-quota.go

@@ -63,7 +63,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 }
 
 func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}

cmd/bucket-replication.go

@@ -24,21 +24,24 @@ import (
 	"strings"
 	"time"
 
+	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/bandwidth"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/event"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -52,12 +55,19 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
 func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
-	clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.RoleArn)
+	arn, err := madmin.ParseARN(rCfg.RoleArn)
+	if err != nil {
+		return false, BucketRemoteArnInvalid{}
+	}
+	if arn.Type != madmin.ReplicationService {
+		return false, BucketRemoteArnTypeInvalid{}
+	}
+	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rCfg.RoleArn)
 	if clnt == nil {
 		return false, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	if found, _ := clnt.BucketExists(ctx, rCfg.GetDestination().Bucket); !found {
-		return false, BucketReplicationDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
+		return false, BucketRemoteDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
 	}
 	if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
 		if ret.LockEnabled {
@@ -82,19 +92,19 @@ func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object strin
 	if permErr != ErrNone {
 		return false
 	}
-	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+	return mustReplicater(ctx, bucket, object, meta, replStatus)
 }
 
 // mustReplicate returns true if object meets replication criteria.
 func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
 		return false
 	}
-	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+	return mustReplicater(ctx, bucket, object, meta, replStatus)
 }
 
 // mustReplicater returns true if object meets replication criteria.
-func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) bool {
 	if globalIsGateway {
 		return false
 	}
@@ -119,7 +129,171 @@ func mustReplicater(ctx context.Context, r *http.Request, bucket, object string,
 	return cfg.Replicate(opts)
 }
 
-func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+// returns true if any of the objects being deleted qualifies for replication.
+func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToDelete) bool {
+	c, err := getReplicationConfig(ctx, bucket)
+	if err != nil || c == nil {
+		return false
+	}
+	for _, obj := range objects {
+		if c.HasActiveRules(obj.ObjectName, true) {
+			return true
+		}
+	}
+	return false
+}
+
+// returns whether object version is a deletemarker and if object qualifies for replication
+func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, gerr error) (dm, replicate bool) {
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil || rcfg == nil {
+		return false, false
+	}
+	// when incoming delete is removal of a delete marker( a.k.a versioned delete),
+	// GetObjectInfo returns extra information even though it returns errFileNotFound
+	if gerr != nil {
+		validReplStatus := false
+		switch oi.ReplicationStatus {
+		case replication.Pending, replication.Complete, replication.Failed:
+			validReplStatus = true
+		}
+		if oi.DeleteMarker && validReplStatus {
+			return oi.DeleteMarker, true
+		}
+		return oi.DeleteMarker, false
+	}
+	opts := replication.ObjectOpts{
+		Name:         dobj.ObjectName,
+		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:     oi.UserTags,
+		DeleteMarker: true,
+		VersionID:    dobj.VersionID,
+	}
+	return oi.DeleteMarker, rcfg.Replicate(opts)
+}
+
+// replicate deletes to the designated replication target if replication configuration
+// has delete marker replication or delete replication (MinIO extension to allow deletes where version id
+// is specified) enabled.
+// Similar to bucket replication for PUT operation, soft delete (a.k.a setting delete marker) and
+// permanent deletes (by specifying a version ID in the delete operation) have three states "Pending", "Complete"
+// and "Failed" to mark the status of the replication of "DELETE" operation. All failed operations can
+// then be retried by healing. In the case of permanent deletes, until the replication is completed on the
+// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
+// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
+// on target.
+func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) {
+	bucket := dobj.Bucket
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil || rcfg == nil {
+		return
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
+	if tgt == nil {
+		return
+	}
+	versionID := dobj.DeleteMarkerVersionID
+	if versionID == "" {
+		versionID = dobj.VersionID
+	}
+	rmErr := tgt.RemoveObject(ctx, rcfg.GetDestination().Bucket, dobj.ObjectName, miniogo.RemoveObjectOptions{
+		VersionID: versionID,
+		Internal: miniogo.AdvancedRemoveOptions{
+			ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "",
+			ReplicationMTime:        dobj.DeleteMarkerMTime.Time,
+			ReplicationStatus:       miniogo.ReplicationStatusReplica,
+		},
+	})
+
+	replicationStatus := dobj.DeleteMarkerReplicationStatus
+	versionPurgeStatus := dobj.VersionPurgeStatus
+
+	if rmErr != nil {
+		if dobj.VersionID == "" {
+			replicationStatus = string(replication.Failed)
+		} else {
+			versionPurgeStatus = Failed
+		}
+	} else {
+		if dobj.VersionID == "" {
+			replicationStatus = string(replication.Complete)
+		} else {
+			versionPurgeStatus = Complete
+		}
+	}
+	var eventName = event.ObjectReplicationComplete
+	if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed {
+		eventName = event.ObjectReplicationFailed
+	}
+	objInfo := ObjectInfo{
+		Name:               dobj.ObjectName,
+		DeleteMarker:       dobj.DeleteMarker,
+		VersionID:          versionID,
+		ReplicationStatus:  replication.StatusType(dobj.DeleteMarkerReplicationStatus),
+		VersionPurgeStatus: versionPurgeStatus,
+	}
+
+	eventArg := &eventArgs{
+		BucketName: bucket,
+		Object:     objInfo,
+		Host:       "Internal: [Replication]",
+		EventName:  eventName,
+	}
+	sendEvent(*eventArg)
+
+	// Update metadata on the delete marker or purge permanent delete if replication success.
+	if _, err = objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
+		VersionID:                     versionID,
+		DeleteMarker:                  dobj.DeleteMarker,
+		DeleteMarkerReplicationStatus: replicationStatus,
+		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
+		VersionPurgeStatus:            versionPurgeStatus,
+		VersionSuspended:              globalBucketVersioningSys.Suspended(bucket),
+	}); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s %s: %w", bucket, dobj.ObjectName, dobj.VersionID, err))
+	}
+}
+
+func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]string {
+	meta := make(map[string]string, len(oi.UserDefined))
+	for k, v := range oi.UserDefined {
+		if k == xhttp.AmzBucketReplicationStatus {
+			continue
+		}
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			continue
+		}
+		meta[k] = v
+	}
+	if oi.ContentEncoding != "" {
+		meta[xhttp.ContentEncoding] = oi.ContentEncoding
+	}
+	if oi.ContentType != "" {
+		meta[xhttp.ContentType] = oi.ContentType
+	}
+	tag, err := tags.ParseObjectTags(oi.UserTags)
+	if err != nil {
+		return nil
+	}
+	if tag != nil {
+		meta[xhttp.AmzObjectTagging] = tag.String()
+		meta[xhttp.AmzTagDirective] = "REPLACE"
+	}
+	sc := dest.StorageClass
+	if sc == "" {
+		sc = oi.StorageClass
+	}
+	meta[xhttp.AmzStorageClass] = sc
+	if oi.UserTags != "" {
+		meta[xhttp.AmzObjectTagging] = oi.UserTags
+	}
+	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
+	meta[xhttp.MinIOSourceETag] = oi.ETag
+	meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
+	return meta
+}
+
+func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
 		if k == xhttp.AmzBucketReplicationStatus {
@@ -139,22 +313,24 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		sc = objInfo.StorageClass
 	}
 	putOpts = miniogo.PutObjectOptions{
-		UserMetadata:         meta,
+		UserMetadata:    meta,
-		UserTags:             tag.ToMap(),
+		UserTags:        tag.ToMap(),
-		ContentType:          objInfo.ContentType,
+		ContentType:     objInfo.ContentType,
-		ContentEncoding:      objInfo.ContentEncoding,
+		ContentEncoding: objInfo.ContentEncoding,
-		StorageClass:         sc,
+		StorageClass:    sc,
-		ReplicationVersionID: objInfo.VersionID,
+		Internal: miniogo.AdvancedPutOptions{
-		ReplicationStatus:    miniogo.ReplicationStatusReplica,
+			SourceVersionID:   objInfo.VersionID,
-		ReplicationMTime:     objInfo.ModTime,
+			ReplicationStatus: miniogo.ReplicationStatusReplica,
-		ReplicationETag:      objInfo.ETag,
+			SourceMTime:       objInfo.ModTime,
+			SourceETag:        objInfo.ETag,
+		},
 	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
 		putOpts.Mode = rmode
 	}
 	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
-		rdate, err := time.Parse(time.RFC3339, retainDateStr)
+		rdate, err := time.Parse(time.RFC3339Nano, retainDateStr)
 		if err != nil {
 			return
 		}
@@ -166,9 +342,57 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
+
 	return
 }
 
+type replicationAction string
+
+const (
+	replicateMetadata replicationAction = "metadata"
+	replicateNone     replicationAction = "none"
+	replicateAll      replicationAction = "all"
+)
+
+// returns replicationAction by comparing metadata between source and target
+func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationAction {
+	// needs full replication
+	if oi1.ETag != oi2.ETag ||
+		oi1.VersionID != oi2.VersionID ||
+		oi1.Size != oi2.Size ||
+		oi1.DeleteMarker != oi2.IsDeleteMarker {
+		return replicateAll
+	}
+
+	if !oi1.ModTime.Equal(oi2.LastModified) ||
+		oi1.ContentType != oi2.ContentType ||
+		oi1.StorageClass != oi2.StorageClass {
+		return replicateMetadata
+	}
+	if oi1.ContentEncoding != "" {
+		enc, ok := oi2.UserMetadata[xhttp.ContentEncoding]
+		if !ok || enc != oi1.ContentEncoding {
+			return replicateMetadata
+		}
+	}
+	for k, v := range oi2.UserMetadata {
+		oi2.Metadata[k] = []string{v}
+	}
+	if len(oi2.Metadata) != len(oi1.UserDefined) {
+		return replicateMetadata
+	}
+	for k1, v1 := range oi1.UserDefined {
+		if v2, ok := oi2.Metadata[k1]; !ok || v1 != strings.Join(v2, "") {
+			return replicateMetadata
+		}
+	}
+	t, _ := tags.MapToObjectTags(oi2.UserTags)
+	if t.String() != oi1.UserTags {
+		return replicateMetadata
+	}
+	return replicateNone
+}
+
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
 func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) {
@@ -180,19 +404,17 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		logger.LogIf(ctx, err)
 		return
 	}
-
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
-	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, cfg.RoleArn))
 		return
 	}
-
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
 		return
 	}
-
 	objInfo = gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
@@ -207,27 +429,47 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		return
 	}
 
-	// if heal encounters a pending replication status, either replication
+	rtype := replicateAll
-	// has failed due to server shutdown or crawler and PutObject replication are in contention.
+	oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
-	healPending := objInfo.ReplicationStatus == replication.Pending
+	if err == nil {
-
+		rtype = getReplicationAction(objInfo, oi)
-	// In the rare event that replication is in pending state either due to
+		if rtype == replicateNone {
-	// server shut down/crash before replication completed or healing and PutObject
-	// race - do an additional stat to see if the version ID exists
-	if healPending {
-		_, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
-		if err == nil {
 			gr.Close()
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed.
 			return
 		}
 	}
-	putOpts := putReplicationOpts(dest, objInfo)
 
+	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+		return
+	}
+	putOpts := putReplicationOpts(ctx, dest, objInfo)
 	replicationStatus := replication.Complete
-	_, err = tgt.PutObject(ctx, dest.Bucket, object, gr, size, "", "", putOpts)
+
-	gr.Close()
+	// Setup bandwidth throttling
+	peers, _ := globalEndpoints.peers()
+	totalNodesCount := len(peers)
+	if totalNodesCount == 0 {
+		totalNodesCount = 1 // For standalone erasure coding
+	}
+	b := target.BandwidthLimit / int64(totalNodesCount)
+	var headerSize int
+	for k, v := range putOpts.Header() {
+		headerSize += len(k) + len(v)
+	}
+	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+	if rtype == replicateAll {
+		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
+	} else {
+		// replicate metadata for object tagging/copy with metadata replacement
+		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
+		_, err = tgt.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
+	}
+
+	r.Close()
 	if err != nil {
 		replicationStatus = replication.Failed
 	}
@@ -240,15 +482,16 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	// - event.ObjectReplicationNotTracked
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
+	var eventName = event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
-		sendEvent(eventArgs{
+		eventName = event.ObjectReplicationFailed
-			EventName:  event.ObjectReplicationFailed,
-			BucketName: bucket,
-			Object:     objInfo,
-			Host:       "Internal: [Replication]",
-		})
 	}
-
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: bucket,
+		Object:     objInfo,
+		Host:       "Internal: [Replication]",
+	})
 	objInfo.metadataOnly = true // Perform only metadata updates.
 	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
 		VersionID: objInfo.VersionID,
@@ -282,18 +525,37 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 	return dst
 }
 
+// DeletedObjectVersionInfo has info on deleted object
+type DeletedObjectVersionInfo struct {
+	DeletedObject
+	Bucket string
+}
 type replicationState struct {
 	// add future metrics here
-	replicaCh chan ObjectInfo
+	replicaCh       chan ObjectInfo
+	replicaDeleteCh chan DeletedObjectVersionInfo
 }
 
 func (r *replicationState) queueReplicaTask(oi ObjectInfo) {
+	if r == nil {
+		return
+	}
 	select {
 	case r.replicaCh <- oi:
 	default:
 	}
 }
 
+func (r *replicationState) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) {
+	if r == nil {
+		return
+	}
+	select {
+	case r.replicaDeleteCh <- doi:
+	default:
+	}
+}
+
 var (
 	globalReplicationState *replicationState
 	// TODO: currently keeping it conservative
@@ -310,11 +572,13 @@ func newReplicationState() *replicationState {
 		globalReplicationConcurrent = 1
 	}
 	rs := &replicationState{
-		replicaCh: make(chan ObjectInfo, globalReplicationConcurrent*2),
+		replicaCh:       make(chan ObjectInfo, 10000),
+		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 10000),
 	}
 	go func() {
 		<-GlobalContext.Done()
 		close(rs.replicaCh)
+		close(rs.replicaDeleteCh)
 	}()
 	return rs
 }
@@ -332,6 +596,11 @@ func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer)
 					return
 				}
 				replicateObject(ctx, oi, objectAPI)
+			case doi, ok := <-r.replicaDeleteCh:
+				if !ok {
+					return
+				}
+				replicateDelete(ctx, doi, objectAPI)
 			}
 		}
 	}()

cmd/bucket-targets.go

@@ -18,14 +18,20 @@ package cmd
 
 import (
 	"context"
+	"encoding/hex"
+	"encoding/json"
 	"net/http"
+	"strings"
 	"sync"
+	"time"
 
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/madmin"
+	sha256 "github.com/minio/sha256-simd"
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -33,7 +39,6 @@ type BucketTargetSys struct {
 	sync.RWMutex
 	arnRemotesMap map[string]*miniogo.Core
 	targetsMap    map[string][]madmin.BucketTarget
-	clientsCache  map[string]*miniogo.Core
 }
 
 // ListTargets lists bucket targets across tenant or for individual bucket, and returns
@@ -75,59 +80,84 @@ func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string
 }
 
 // SetTarget - sets a new minio-go client target for this bucket.
-func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget) error {
+func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
 	if globalIsGateway {
 		return nil
 	}
-	if !tgt.Type.IsValid() {
+	if !tgt.Type.IsValid() && !update {
 		return BucketRemoteArnTypeInvalid{Bucket: bucket}
 	}
 	clnt, err := sys.getRemoteTargetClient(tgt)
 	if err != nil {
 		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 	}
+	// validate if target credentials are ok
+	if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
+		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
+			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		}
+		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
+	}
 	if tgt.Type == madmin.ReplicationService {
+		if !globalIsErasure {
+			return NotImplemented{}
+		}
 		if !globalBucketVersioningSys.Enabled(bucket) {
 			return BucketReplicationSourceNotVersioned{Bucket: bucket}
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
-				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
-			}
 			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
-			return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
+		}
+	}
+	if tgt.Type == madmin.ILMService {
+		if globalBucketVersioningSys.Enabled(bucket) {
+			vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
+			if err != nil {
+				if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
+					return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+				}
+				return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
+			}
+			if vcfg.Status != string(versioning.Enabled) {
+				return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
+			}
 		}
 	}
-
 	sys.Lock()
 	defer sys.Unlock()
 
 	tgts := sys.targetsMap[bucket]
 	newtgts := make([]madmin.BucketTarget, len(tgts))
+	labels := make(map[string]struct{})
 	found := false
 	for idx, t := range tgts {
+		labels[t.Label] = struct{}{}
 		if t.Type == tgt.Type {
-			if t.Arn == tgt.Arn {
+			if t.Arn == tgt.Arn && !update {
 				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
 			}
+			if t.Label == tgt.Label && !update {
+				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
+			}
 			newtgts[idx] = *tgt
 			found = true
 			continue
 		}
 		newtgts[idx] = t
 	}
-	if !found {
+	if _, ok := labels[tgt.Label]; ok && !update {
+		return BucketRemoteLabelInUse{Bucket: tgt.TargetBucket}
+	}
+	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
 
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
-	if _, ok := sys.clientsCache[clnt.EndpointURL().String()]; !ok {
-		sys.clientsCache[clnt.EndpointURL().String()] = clnt
-	}
 	return nil
 }
 
@@ -144,6 +174,9 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteArnInvalid{Bucket: bucket}
 	}
 	if arn.Type == madmin.ReplicationService {
+		if !globalIsErasure {
+			return NotImplemented{}
+		}
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
 		if err == nil && rcfg.RoleArn == arnStr {
@@ -152,6 +185,16 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 			}
 		}
 	}
+	if arn.Type == madmin.ILMService {
+		// reject removal of remote target if lifecycle transition uses this arn
+		config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+		if err == nil && transitionSCInUse(ctx, config, bucket, arnStr) {
+			if _, ok := sys.arnRemotesMap[arnStr]; ok {
+				return BucketRemoteRemoveDisallowed{Bucket: bucket}
+			}
+		}
+	}
+
 	// delete ARN type from list of matching targets
 	sys.Lock()
 	defer sys.Unlock()
@@ -173,19 +216,56 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	return nil
 }
 
-// GetReplicationTargetClient returns minio-go client for replication target instance
+// GetRemoteTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetReplicationTargetClient(ctx context.Context, arn string) *miniogo.Core {
+func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
 	sys.RLock()
 	defer sys.RUnlock()
 	return sys.arnRemotesMap[arn]
 }
 
+// GetRemoteTargetWithLabel returns bucket target given a target label
+func (sys *BucketTargetSys) GetRemoteTargetWithLabel(ctx context.Context, bucket, targetLabel string) *madmin.BucketTarget {
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, t := range sys.targetsMap[bucket] {
+		if strings.ToUpper(t.Label) == strings.ToUpper(targetLabel) {
+			tgt := t.Clone()
+			return &tgt
+		}
+	}
+	return nil
+}
+
+// GetRemoteArnWithLabel returns bucket target's ARN given its target label
+func (sys *BucketTargetSys) GetRemoteArnWithLabel(ctx context.Context, bucket, tgtLabel string) *madmin.ARN {
+	tgt := sys.GetRemoteTargetWithLabel(ctx, bucket, tgtLabel)
+	if tgt == nil {
+		return nil
+	}
+	arn, err := madmin.ParseARN(tgt.Arn)
+	if err != nil {
+		return nil
+	}
+	return arn
+}
+
+// GetRemoteLabelWithArn returns a bucket target's label given its ARN
+func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, arnStr string) string {
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, t := range sys.targetsMap[bucket] {
+		if t.Arn == arnStr {
+			return t.Label
+		}
+	}
+	return ""
+}
+
 // NewBucketTargetSys - creates new replication system.
 func NewBucketTargetSys() *BucketTargetSys {
 	return &BucketTargetSys{
 		arnRemotesMap: make(map[string]*miniogo.Core),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
-		clientsCache:  make(map[string]*miniogo.Core),
 	}
 }
 
@@ -205,14 +285,14 @@ func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objA
 	return nil
 }
 
-// UpdateTarget updates target to reflect metadata updates
+// UpdateAllTargets updates target to reflect metadata updates
-func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTargets) {
+func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
 	if sys == nil {
 		return
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if cfg == nil || cfg.Empty() {
+	if tgts == nil || tgts.Empty() {
 		// remove target and arn association
 		if tgts, ok := sys.targetsMap[bucket]; ok {
 			for _, t := range tgts {
@@ -223,20 +303,17 @@ func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTarget
 		return
 	}
 
-	if len(cfg.Targets) > 0 {
+	if len(tgts.Targets) > 0 {
-		sys.targetsMap[bucket] = cfg.Targets
+		sys.targetsMap[bucket] = tgts.Targets
 	}
-	for _, tgt := range cfg.Targets {
+	for _, tgt := range tgts.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = tgtClient
-		if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
-			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
-		}
 	}
-	sys.targetsMap[bucket] = cfg.Targets
+	sys.targetsMap[bucket] = tgts.Targets
 }
 
 // create minio-go clients for buckets having remote targets
@@ -258,9 +335,6 @@ func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objA
 				continue
 			}
 			sys.arnRemotesMap[tgt.Arn] = tgtClient
-			if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
-				sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
-			}
 		}
 		sys.targetsMap[bucket.Name] = cfg.Targets
 	}
@@ -272,16 +346,14 @@ var getRemoteTargetInstanceTransportOnce sync.Once
 
 // Returns a minio-go Client configured to access remote host described in replication target config.
 func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
-	if clnt, ok := sys.clientsCache[tcfg.Endpoint]; ok {
-		return clnt, nil
-	}
 	config := tcfg.Credentials
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
 
 	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = NewGatewayHTTPTransport()
+		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
 	})
-	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
+
+	core, err := miniogo.NewCore(tcfg.URL().Host, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
 		Transport: getRemoteTargetInstanceTransport,
@@ -296,18 +368,57 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 	}
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
-		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL() == tgt.URL() {
+		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() {
 			return tgt.Arn
 		}
 	}
 	if !madmin.ServiceType(target.Type).IsValid() {
 		return ""
 	}
+	return generateARN(target)
+}
+
+// generate ARN that is unique to this target type
+func generateARN(t *madmin.BucketTarget) string {
+	hash := sha256.New()
+	hash.Write([]byte(t.Type))
+	hash.Write([]byte(t.Region))
+	hash.Write([]byte(t.TargetBucket))
+	hashSum := hex.EncodeToString(hash.Sum(nil))
 	arn := madmin.ARN{
-		Type:   target.Type,
+		Type:   t.Type,
-		ID:     mustGetUUID(),
+		ID:     hashSum,
-		Region: target.Region,
+		Region: t.Region,
-		Bucket: target.TargetBucket,
+		Bucket: t.TargetBucket,
 	}
 	return arn.String()
 }
+
+// Returns parsed target config. If KMS is configured, remote target is decrypted
+func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) {
+	var (
+		data []byte
+		err  error
+		t    madmin.BucketTargets
+		meta map[string]string
+	)
+	if len(cdata) == 0 {
+		return nil, nil
+	}
+	data = cdata
+	if len(cmetadata) != 0 {
+		if err := json.Unmarshal(cmetadata, &meta); err != nil {
+			return nil, err
+		}
+		if crypto.S3.IsEncrypted(meta) {
+			if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{bucket: bucket, bucketTargetsFile: bucketTargetsFile}); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if err = json.Unmarshal(data, &t); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}

cmd/common-main.go

@@ -21,6 +21,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"net/url"
 	"os"
@@ -29,20 +30,36 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/color"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
 )
 
+// serverDebugLog will enable debug printing
+var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+
 func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
+	rand.Seed(time.Now().UTC().UnixNano())
+	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 3*time.Second)
+
+	initGlobalContext()
+
+	globalReplicationState = newReplicationState()
+	globalTransitionState = newTransitionState()
+
+	console.SetColor("Debug", color.New())
+
 	gob.Register(StorageErr(""))
 }
 
@@ -59,10 +76,12 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 		}
 	}
 
+	globalCompressConfigMu.Lock()
 	if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
 		logger.Fatal(errInvalidArgument,
 			"Compression support is requested but '%s' does not support compression", name)
 	}
+	globalCompressConfigMu.Unlock()
 }
 
 // Check for updates and print a notification message
@@ -100,11 +119,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	if globalInplaceUpdateDisabled {
+	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
-		logStartupMessage(updateMsg)
-	} else {
-		logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
-	}
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {

cmd/config-current.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
@@ -28,6 +29,7 @@ import (
 	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
+	"github.com/minio/minio/cmd/config/heal"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -56,6 +58,7 @@ func initHelp() {
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
+		config.HealSubSys:           heal.DefaultKVS,
 		config.CrawlerSubSys:        crawler.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
@@ -108,9 +111,13 @@ func initHelp() {
 			Key:         config.APISubSys,
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
+		config.HelpKV{
+			Key:         config.HealSubSys,
+			Description: "manage object healing frequency and bitrot verification checks",
+		},
 		config.HelpKV{
 			Key:         config.CrawlerSubSys,
-			Description: "manage continuous disk crawling for bucket disk usage, lifecycle, quota and data integrity checks",
+			Description: "manage crawling for usage calculation, lifecycle, healing and more",
 		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
@@ -191,6 +198,7 @@ func initHelp() {
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
+		config.HealSubSys:           heal.Help,
 		config.CrawlerSubSys:        crawler.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
@@ -221,6 +229,9 @@ var (
 )
 
 func validateConfig(s config.Config, setDriveCount int) error {
+	// We must have a global lock for this so nobody else modifies env while we do.
+	defer env.LockSetEnv()()
+
 	// Disable merging env values with config for validation.
 	env.SetEnvOff()
 
@@ -249,7 +260,18 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
-	if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
+	compCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+	if err != nil {
+		return err
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI != nil {
+		if compCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
+	}
+
+	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -438,10 +460,6 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
-	globalCrawlerConfig, err = crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to read crawler config: %w", err))
-	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
@@ -459,11 +477,6 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
 	}
 
-	globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup Compression: %w", err))
-	}
-
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
@@ -538,6 +551,68 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
+
+	// Apply dynamic config values
+	logger.LogIf(ctx, applyDynamicConfig(ctx, s))
+}
+
+// applyDynamicConfig will apply dynamic config values.
+// Dynamic systems should be in config.SubSystemsDynamic as well.
+func applyDynamicConfig(ctx context.Context, s config.Config) error {
+	// Read all dynamic configs.
+	// API
+	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+	}
+
+	// Compression
+	cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+	if err != nil {
+		return fmt.Errorf("Unable to setup Compression: %w", err)
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI != nil {
+		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
+	}
+
+	// Heal
+	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to apply heal config: %w", err))
+	}
+
+	// Crawler
+	crawlerCfg, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	if err != nil {
+		return fmt.Errorf("Unable to apply crawler config: %w", err)
+	}
+
+	// Apply configurations.
+	// We should not fail after this.
+	globalAPIConfig.init(apiConfig, globalAPIConfig.setDriveCount)
+
+	globalCompressConfigMu.Lock()
+	globalCompressConfig = cmpCfg
+	globalCompressConfigMu.Unlock()
+
+	globalHealConfigMu.Lock()
+	globalHealConfig = healCfg
+	globalHealConfigMu.Unlock()
+
+	logger.LogIf(ctx, crawlerSleeper.Update(crawlerCfg.Delay, crawlerCfg.MaxWait))
+
+	// Update all dynamic config values in memory.
+	globalServerConfigMu.Lock()
+	defer globalServerConfigMu.Unlock()
+	if globalServerConfig != nil {
+		for k := range config.SubSystemsDynamic {
+			globalServerConfig[k] = s[k]
+		}
+	}
+	return nil
 }
 
 // Help - return sub-system level help

cmd/config-dir.go

@@ -87,7 +87,7 @@ func mkdirAllIgnorePerm(path string) error {
 	if err != nil {
 		// It is possible in kubernetes like deployments this directory
 		// is already mounted and is not writable, ignore any write errors.
-		if os.IsPermission(err) {
+		if osIsPermission(err) {
 			err = nil
 		}
 	}

cmd/config-encrypted.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"time"
 	"unicode/utf8"
@@ -28,7 +27,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/madmin"
-	etcd "go.etcd.io/etcd/v3/clientv3"
+	etcd "go.etcd.io/etcd/clientv3"
 )
 
 func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
@@ -194,7 +193,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 				// Config is already encrypted with right keys
 				continue
 			}
-			return errors.New("config data not in plain-text form or encrypted")
+			return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -274,7 +273,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 					// Config is already encrypted with right keys
 					continue
 				}
-				return errors.New("config data not in plain-text form or encrypted")
+				return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)

cmd/config-migrate.go

@@ -74,7 +74,7 @@ func migrateConfig() error {
 	// Load only config version information.
 	version, err := GetVersion(getConfigFile())
 	if err != nil {
-		if os.IsNotExist(err) || os.IsPermission(err) {
+		if osIsNotExist(err) || osIsPermission(err) {
 			return nil
 		}
 		return err
@@ -243,7 +243,7 @@ func purgeV1() error {
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %w", err)
@@ -264,7 +264,7 @@ func migrateV2ToV3() error {
 
 	cv2 := &configV2{}
 	_, err := Load(configFile, cv2)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘2’. %w", err)
@@ -323,7 +323,7 @@ func migrateV3ToV4() error {
 
 	cv3 := &configV3{}
 	_, err := Load(configFile, cv3)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘3’. %w", err)
@@ -361,7 +361,7 @@ func migrateV4ToV5() error {
 
 	cv4 := &configV4{}
 	_, err := Load(configFile, cv4)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘4’. %w", err)
@@ -402,7 +402,7 @@ func migrateV5ToV6() error {
 
 	cv5 := &configV5{}
 	_, err := Load(configFile, cv5)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘5’. %w", err)
@@ -491,7 +491,7 @@ func migrateV6ToV7() error {
 
 	cv6 := &configV6{}
 	_, err := Load(configFile, cv6)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘6’. %w", err)
@@ -547,7 +547,7 @@ func migrateV7ToV8() error {
 
 	cv7 := &serverConfigV7{}
 	_, err := Load(configFile, cv7)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘7’. %w", err)
@@ -609,7 +609,7 @@ func migrateV8ToV9() error {
 
 	cv8 := &serverConfigV8{}
 	_, err := Load(configFile, cv8)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘8’. %w", err)
@@ -679,7 +679,7 @@ func migrateV9ToV10() error {
 
 	cv9 := &serverConfigV9{}
 	_, err := Load(configFile, cv9)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘9’. %w", err)
@@ -747,7 +747,7 @@ func migrateV10ToV11() error {
 
 	cv10 := &serverConfigV10{}
 	_, err := Load(configFile, cv10)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘10’. %w", err)
@@ -818,7 +818,7 @@ func migrateV11ToV12() error {
 
 	cv11 := &serverConfigV11{}
 	_, err := Load(configFile, cv11)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘11’. %w", err)
@@ -915,7 +915,7 @@ func migrateV12ToV13() error {
 
 	cv12 := &serverConfigV12{}
 	_, err := Load(configFile, cv12)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘12’. %w", err)
@@ -995,7 +995,7 @@ func migrateV13ToV14() error {
 
 	cv13 := &serverConfigV13{}
 	_, err := Load(configFile, cv13)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘13’. %w", err)
@@ -1080,7 +1080,7 @@ func migrateV14ToV15() error {
 
 	cv14 := &serverConfigV14{}
 	_, err := Load(configFile, cv14)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘14’. %w", err)
@@ -1170,7 +1170,7 @@ func migrateV15ToV16() error {
 
 	cv15 := &serverConfigV15{}
 	_, err := Load(configFile, cv15)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘15’. %w", err)
@@ -1260,7 +1260,7 @@ func migrateV16ToV17() error {
 
 	cv16 := &serverConfigV16{}
 	_, err := Load(configFile, cv16)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘16’. %w", err)
@@ -1381,7 +1381,7 @@ func migrateV17ToV18() error {
 
 	cv17 := &serverConfigV17{}
 	_, err := Load(configFile, cv17)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘17’. %w", err)
@@ -1483,7 +1483,7 @@ func migrateV18ToV19() error {
 
 	cv18 := &serverConfigV18{}
 	_, err := Load(configFile, cv18)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %w", err)
@@ -1589,7 +1589,7 @@ func migrateV19ToV20() error {
 
 	cv19 := &serverConfigV19{}
 	_, err := Load(configFile, cv19)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %w", err)
@@ -1694,7 +1694,7 @@ func migrateV20ToV21() error {
 
 	cv20 := &serverConfigV20{}
 	_, err := Load(configFile, cv20)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘20’. %w", err)
@@ -1798,7 +1798,7 @@ func migrateV21ToV22() error {
 
 	cv21 := &serverConfigV21{}
 	_, err := Load(configFile, cv21)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘21’. %w", err)
@@ -1902,7 +1902,7 @@ func migrateV22ToV23() error {
 
 	cv22 := &serverConfigV22{}
 	_, err := Load(configFile, cv22)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘22’. %w", err)
@@ -2015,7 +2015,7 @@ func migrateV23ToV24() error {
 
 	cv23 := &serverConfigV23{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘23’. %w", err)
@@ -2128,7 +2128,7 @@ func migrateV24ToV25() error {
 
 	cv24 := &serverConfigV24{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘24’. %w", err)
@@ -2246,7 +2246,7 @@ func migrateV25ToV26() error {
 
 	cv25 := &serverConfigV25{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘25’. %w", err)
@@ -2368,7 +2368,7 @@ func migrateV26ToV27() error {
 	// in the new `logger` field
 	srvConfig := &serverConfigV27{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
@@ -2401,7 +2401,7 @@ func migrateV27ToV28() error {
 
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
@@ -2452,17 +2452,17 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	var config = &serverConfigV27{}
 	for _, cfgFile := range configFiles {
 		if _, err = Load(cfgFile, config); err != nil {
-			if !os.IsNotExist(err) && !os.IsPermission(err) {
+			if !osIsNotExist(err) && !osIsPermission(err) {
 				return err
 			}
 			continue
 		}
 		break
 	}
-	if os.IsPermission(err) {
+	if osIsPermission(err) {
 		logger.Info("Older config found but not readable %s, proceeding to initialize new config anyways", err)
 	}
-	if os.IsNotExist(err) || os.IsPermission(err) {
+	if osIsNotExist(err) || osIsPermission(err) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}

cmd/config-migrate_test.go

@@ -60,7 +60,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	}
 
 	// Check if config v1 is removed from filesystem
-	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
+	if _, err := os.Stat(configPath); err == nil || !osIsNotExist(err) {
 		t.Fatal("Config V1 file is not purged")
 	}
 

cmd/config/api/api.go

@@ -34,12 +34,17 @@ const (
 	apiClusterDeadline         = "cluster_deadline"
 	apiCorsAllowOrigin         = "cors_allow_origin"
 	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiExtendListCacheLife     = "extend_list_cache_life"
 
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
 	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
 	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
+	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
+	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
+	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
 )
 
 // Deprecated key and ENVs
@@ -71,6 +76,14 @@ var (
 			Key:   apiRemoteTransportDeadline,
 			Value: "2h",
 		},
+		config.KV{
+			Key:   apiListQuorum,
+			Value: "optimal",
+		},
+		config.KV{
+			Key:   apiExtendListCacheLife,
+			Value: "0s",
+		},
 	}
 )
 
@@ -81,6 +94,8 @@ type Config struct {
 	ClusterDeadline         time.Duration `json:"cluster_deadline"`
 	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
+	ListQuorum              string        `json:"list_strict_quorum"`
+	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -94,6 +109,22 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, &aux)
 }
 
+// GetListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func (sCfg Config) GetListQuorum() int {
+	switch sCfg.ListQuorum {
+	case "reduced":
+		return 2
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "strict":
+		return -1
+	}
+	// Defaults to 3 drives per set, defaults to "optimal" value
+	return 3
+}
+
 // LookupConfig - lookup api config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	// remove this since we have removed this already.
@@ -130,11 +161,25 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	listQuorum := env.Get(EnvAPIListQuorum, kvs.Get(apiListQuorum))
+	switch listQuorum {
+	case "strict", "optimal", "reduced", "disk":
+	default:
+		return cfg, errors.New("invalid value for list strict quorum")
+	}
+
+	listLife, err := time.ParseDuration(env.Get(EnvAPIExtendListCacheLife, kvs.Get(apiExtendListCacheLife)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
 		ClusterDeadline:         clusterDeadline,
 		CorsAllowOrigin:         corsAllowOrigin,
 		RemoteTransportDeadline: remoteTransportDeadline,
+		ListQuorum:              listQuorum,
+		ExtendListLife:          listLife,
 	}, nil
 }

cmd/config/cache/config.go

@@ -28,16 +28,17 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled       bool     `json:"-"`
+	Enabled         bool     `json:"-"`
-	Drives        []string `json:"drives"`
+	Drives          []string `json:"drives"`
-	Expiry        int      `json:"expiry"`
+	Expiry          int      `json:"expiry"`
-	MaxUse        int      `json:"maxuse"`
+	MaxUse          int      `json:"maxuse"`
-	Quota         int      `json:"quota"`
+	Quota           int      `json:"quota"`
-	Exclude       []string `json:"exclude"`
+	Exclude         []string `json:"exclude"`
-	After         int      `json:"after"`
+	After           int      `json:"after"`
-	WatermarkLow  int      `json:"watermark_low"`
+	WatermarkLow    int      `json:"watermark_low"`
-	WatermarkHigh int      `json:"watermark_high"`
+	WatermarkHigh   int      `json:"watermark_high"`
-	Range         bool     `json:"range"`
+	Range           bool     `json:"range"`
+	CommitWriteback bool     `json:"-"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -152,3 +153,13 @@ func parseCacheExcludes(excludes string) ([]string, error) {
 
 	return excludesSlice, nil
 }
+
+func parseCacheCommitMode(commitStr string) (bool, error) {
+	switch strings.ToLower(commitStr) {
+	case "writeback":
+		return true, nil
+	case "writethrough":
+		return false, nil
+	}
+	return false, config.ErrInvalidCacheCommitValue(nil).Msg("cache commit value must be `writeback` or `writethrough`")
+}

cmd/config/cache/help.go

@@ -74,5 +74,11 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         Commit,
+			Description: `set to control cache commit behavior, defaults to "writethrough"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -35,6 +35,7 @@ const (
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
 	Range         = "range"
+	Commit        = "commit"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -45,6 +46,7 @@ const (
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 	EnvCacheRange         = "MINIO_CACHE_RANGE"
+	EnvCacheCommit        = "MINIO_CACHE_COMMIT"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -53,6 +55,7 @@ const (
 	DefaultAfter         = "0"
 	DefaultWaterMarkLow  = "70"
 	DefaultWaterMarkHigh = "80"
+	DefaultCacheCommit   = "writethrough"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -90,6 +93,10 @@ var (
 			Key:   Range,
 			Value: config.EnableOn,
 		},
+		config.KV{
+			Key:   Commit,
+			Value: DefaultCacheCommit,
+		},
 	}
 )
 
@@ -210,6 +217,16 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		}
 		cfg.Range = rng
 	}
+	if commit := env.Get(EnvCacheCommit, kvs.Get(Commit)); commit != "" {
+		cfg.CommitWriteback, err = parseCacheCommitMode(commit)
+		if err != nil {
+			return cfg, err
+		}
+		if cfg.After > 0 && cfg.CommitWriteback {
+			err := errors.New("cache after cannot be used with commit writeback")
+			return cfg, config.ErrInvalidCacheSetting(err)
+		}
+	}
 
 	return cfg, nil
 }

cmd/config/config.go

@@ -76,6 +76,7 @@ const (
 	KmsKesSubSys         = "kms_kes"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
+	HealSubSys           = "heal"
 	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
@@ -98,7 +99,7 @@ const (
 )
 
 // SubSystems - all supported sub-systems
-var SubSystems = set.CreateStringSet([]string{
+var SubSystems = set.CreateStringSet(
 	CredentialsSubSys,
 	RegionSubSys,
 	EtcdSubSys,
@@ -114,6 +115,7 @@ var SubSystems = set.CreateStringSet([]string{
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
 	CrawlerSubSys,
+	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
 	NotifyKafkaSubSys,
@@ -124,7 +126,15 @@ var SubSystems = set.CreateStringSet([]string{
 	NotifyPostgresSubSys,
 	NotifyRedisSubSys,
 	NotifyWebhookSubSys,
-}...)
+)
+
+// SubSystemsDynamic - all sub-systems that have dynamic config.
+var SubSystemsDynamic = set.CreateStringSet(
+	APISubSys,
+	CompressionSubSys,
+	CrawlerSubSys,
+	HealSubSys,
+)
 
 // SubSystemsSingleTargets - subsystems which only support single target.
 var SubSystemsSingleTargets = set.CreateStringSet([]string{
@@ -140,6 +150,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	HealSubSys,
 	CrawlerSubSys,
 }...)
 
@@ -309,25 +320,29 @@ func (c Config) DelFrom(r io.Reader) error {
 	return nil
 }
 
-// ReadFrom - implements io.ReaderFrom interface
+// ReadConfig - read content from input and write into c.
-func (c Config) ReadFrom(r io.Reader) (int64, error) {
+// Returns whether all parameters were dynamic.
+func (c Config) ReadConfig(r io.Reader) (dynOnly bool, err error) {
 	var n int
 	scanner := bufio.NewScanner(r)
+	dynOnly = true
 	for scanner.Scan() {
 		// Skip any empty lines, or comment like characters
 		text := scanner.Text()
 		if text == "" || strings.HasPrefix(text, KvComment) {
 			continue
 		}
-		if err := c.SetKVS(text, DefaultKVS); err != nil {
+		dynamic, err := c.SetKVS(text, DefaultKVS)
-			return 0, err
+		if err != nil {
+			return false, err
 		}
+		dynOnly = dynOnly && dynamic
 		n += len(text)
 	}
 	if err := scanner.Err(); err != nil {
-		return 0, err
+		return false, err
 	}
-	return int64(n), nil
+	return dynOnly, nil
 }
 
 type configWriteTo struct {
@@ -618,26 +633,27 @@ func (c Config) Clone() Config {
 }
 
 // SetKVS - set specific key values per sub-system.
-func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
+func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) {
 	if len(s) == 0 {
-		return Errorf("input arguments cannot be empty")
+		return false, Errorf("input arguments cannot be empty")
 	}
 	inputs := strings.SplitN(s, KvSpaceSeparator, 2)
 	if len(inputs) <= 1 {
-		return Errorf("invalid number of arguments '%s'", s)
+		return false, Errorf("invalid number of arguments '%s'", s)
 	}
 	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
 	if len(subSystemValue) == 0 {
-		return Errorf("invalid number of arguments %s", s)
+		return false, Errorf("invalid number of arguments %s", s)
 	}
 
 	if !SubSystems.Contains(subSystemValue[0]) {
-		return Errorf("unknown sub-system %s", s)
+		return false, Errorf("unknown sub-system %s", s)
 	}
 
 	if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 {
-		return Errorf("sub-system '%s' only supports single target", subSystemValue[0])
+		return false, Errorf("sub-system '%s' only supports single target", subSystemValue[0])
 	}
+	dynamic = SubSystemsDynamic.Contains(subSystemValue[0])
 
 	tgt := Default
 	subSys := subSystemValue[0]
@@ -647,7 +663,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 
 	fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
 	if len(fields) == 0 {
-		return Errorf("sub-system '%s' cannot have empty keys", subSys)
+		return false, Errorf("sub-system '%s' cannot have empty keys", subSys)
 	}
 
 	var kvs = KVS{}
@@ -670,7 +686,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 			kvs.Set(prevK, madmin.SanitizeValue(kv[1]))
 			continue
 		}
-		return Errorf("key '%s', cannot have empty value", kv[0])
+		return false, Errorf("key '%s', cannot have empty value", kv[0])
 	}
 
 	_, ok := kvs.Lookup(Enable)
@@ -720,11 +736,11 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 			// Return error only if the
 			// key is enabled, for state=off
 			// let it be empty.
-			return Errorf(
+			return false, Errorf(
 				"'%s' is not optional for '%s' sub-system, please check '%s' documentation",
 				hkv.Key, subSys, subSys)
 		}
 	}
 	c[subSys][tgt] = currKVS
-	return nil
+	return dynamic, nil
 }

cmd/config/crawler/crawler.go

@@ -17,38 +17,56 @@
 package crawler
 
 import (
-	"errors"
+	"strconv"
+	"time"
 
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
 )
 
 // Compression environment variables
 const (
-	BitrotScan = "bitrotscan"
+	Delay   = "delay"
+	MaxWait = "max_wait"
+
+	EnvDelay   = "MINIO_CRAWLER_DELAY"
+	EnvMaxWait = "MINIO_CRAWLER_MAX_WAIT"
 )
 
-// Config represents the crawler settings.
+// Config represents the heal settings.
 type Config struct {
-	// Bitrot will perform bitrot scan on local disk when checking objects.
+	// Delay is the sleep multiplier.
-	Bitrot bool `json:"bitrotscan"`
+	Delay float64 `json:"delay"`
+	// MaxWait is maximum wait time between operations
+	MaxWait time.Duration
 }
 
 var (
-	// DefaultKVS - default KV config for crawler settings
+	// DefaultKVS - default KV config for heal settings
 	DefaultKVS = config.KVS{
 		config.KV{
-			Key:   BitrotScan,
+			Key:   Delay,
-			Value: config.EnableOff,
+			Value: "10",
+		},
+		config.KV{
+			Key:   MaxWait,
+			Value: "15s",
 		},
 	}
 
 	// Help provides help for config values
 	Help = config.HelpKVS{
 		config.HelpKV{
-			Key:         BitrotScan,
+			Key:         Delay,
-			Description: `perform bitrot scan on disks when checking objects during crawl`,
+			Description: `crawler delay multiplier, defaults to '10.0'`,
 			Optional:    true,
-			Type:        "on|off",
+			Type:        "float",
+		},
+		config.HelpKV{
+			Key:         MaxWait,
+			Description: `maximum wait time between operations, defaults to '15s'`,
+			Optional:    true,
+			Type:        "duration",
 		},
 	}
 )
@@ -58,10 +76,13 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
-	bitrot := kvs.Get(BitrotScan)
+	cfg.Delay, err = strconv.ParseFloat(env.Get(EnvDelay, kvs.Get(Delay)), 64)
-	if bitrot != config.EnableOn && bitrot != config.EnableOff {
+	if err != nil {
-		return cfg, errors.New(BitrotScan + ": must be 'on' or 'off'")
+		return cfg, err
+	}
+	cfg.MaxWait, err = time.ParseDuration(env.Get(EnvMaxWait, kvs.Get(MaxWait)))
+	if err != nil {
+		return cfg, err
 	}
-	cfg.Bitrot = bitrot == config.EnableOn
 	return cfg, nil
 }

cmd/config/dns/etcd_dns.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/coredns/coredns/plugin/etcd/msg"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/clientv3"
 )
 
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)

cmd/config/errors.go

@@ -102,6 +102,17 @@ var (
 		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
 	)
 
+	ErrInvalidCacheCommitValue = newErrFn(
+		"Invalid cache commit value",
+		"Please check the passed value",
+		"MINIO_CACHE_COMMIT: Valid expected value is `writeback` or `writethrough`",
+	)
+
+	ErrInvalidCacheSetting = newErrFn(
+		"Incompatible cache setting",
+		"Please check the passed value",
+		"MINIO_CACHE_AFTER cannot be used with MINIO_CACHE_COMMIT setting",
+	)
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/config/etcd/etcd.go

@@ -25,8 +25,8 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
-	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/clientv3"
-	"go.etcd.io/etcd/v3/clientv3/namespace"
+	"go.etcd.io/etcd/clientv3/namespace"
 )
 
 const (

cmd/config/heal/heal.go

@@ -0,0 +1,106 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package heal
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Compression environment variables
+const (
+	Bitrot  = "bitrotscan"
+	Sleep   = "max_sleep"
+	IOCount = "max_io"
+
+	EnvBitrot  = "MINIO_HEAL_BITROTSCAN"
+	EnvSleep   = "MINIO_HEAL_MAX_SLEEP"
+	EnvIOCount = "MINIO_HEAL_MAX_IO"
+)
+
+// Config represents the heal settings.
+type Config struct {
+	// Bitrot will perform bitrot scan on local disk when checking objects.
+	Bitrot bool `json:"bitrotscan"`
+	// maximum sleep duration between objects to slow down heal operation.
+	Sleep   time.Duration `json:"sleep"`
+	IOCount int           `json:"iocount"`
+}
+
+var (
+	// DefaultKVS - default KV config for heal settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   Bitrot,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   Sleep,
+			Value: "1s",
+		},
+		config.KV{
+			Key:   IOCount,
+			Value: "10",
+		},
+	}
+
+	// Help provides help for config values
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Bitrot,
+			Description: `perform bitrot scan on disks when checking objects during crawl`,
+			Optional:    true,
+			Type:        "on|off",
+		},
+		config.HelpKV{
+			Key:         Sleep,
+			Description: `maximum sleep duration between objects to slow down heal operation. eg. 2s`,
+			Optional:    true,
+			Type:        "duration",
+		},
+		config.HelpKV{
+			Key:         IOCount,
+			Description: `maximum IO requests allowed between objects to slow down heal operation. eg. 3`,
+			Optional:    true,
+			Type:        "int",
+		},
+	}
+)
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+	cfg.Bitrot, err = config.ParseBool(env.Get(EnvBitrot, kvs.Get(Bitrot)))
+	if err != nil {
+		return cfg, fmt.Errorf("'heal:bitrotscan' value invalid: %w", err)
+	}
+	cfg.Sleep, err = time.ParseDuration(env.Get(EnvSleep, kvs.Get(Sleep)))
+	if err != nil {
+		return cfg, fmt.Errorf("'heal:max_sleep' value invalid: %w", err)
+	}
+	cfg.IOCount, err = strconv.Atoi(env.Get(EnvIOCount, kvs.Get(IOCount)))
+	if err != nil {
+		return cfg, fmt.Errorf("'heal:max_io' value invalid: %w", err)
+	}
+	return cfg, nil
+}

cmd/config/storageclass/help.go

@@ -33,6 +33,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         ClassDMA,
+			Description: `enable O_DIRECT for both read and write, defaults to "write" e.g. "read+write"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/storageclass/storage-class.go

@@ -18,6 +18,7 @@ package storageclass
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -32,17 +33,26 @@ const (
 	RRS = "REDUCED_REDUNDANCY"
 	// Standard storage class
 	STANDARD = "STANDARD"
+	// DMA storage class
+	DMA = "DMA"
+
+	// Valid values are "write" and "read+write"
+	DMAWrite     = "write"
+	DMAReadWrite = "read+write"
 )
 
 // Standard constats for config info storage class
 const (
 	ClassStandard = "standard"
 	ClassRRS      = "rrs"
+	ClassDMA      = "dma"
 
 	// Reduced redundancy storage class environment variable
 	RRSEnv = "MINIO_STORAGE_CLASS_RRS"
 	// Standard storage class environment variable
 	StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
+	// DMA storage class environment variable
+	DMAEnv = "MINIO_STORAGE_CLASS_DMA"
 
 	// Supported storage class scheme is EC
 	schemePrefix = "EC"
@@ -52,6 +62,9 @@ const (
 
 	// Default RRS parity is always minimum parity.
 	defaultRRSParity = minParityDisks
+
+	// Default DMA value
+	defaultDMA = DMAWrite
 )
 
 // DefaultKVS - default storage class config
@@ -65,18 +78,24 @@ var (
 			Key:   ClassRRS,
 			Value: "EC:2",
 		},
+		config.KV{
+			Key:   ClassDMA,
+			Value: defaultDMA,
+		},
 	}
 )
 
 // StorageClass - holds storage class information
 type StorageClass struct {
 	Parity int
+	DMA    string
 }
 
 // Config storage class configuration
 type Config struct {
 	Standard StorageClass `json:"standard"`
 	RRS      StorageClass `json:"rrs"`
+	DMA      StorageClass `json:"dma"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -93,7 +112,7 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // IsValid - returns true if input string is a valid
 // storage class kind supported.
 func IsValid(sc string) bool {
-	return sc == RRS || sc == STANDARD
+	return sc == RRS || sc == STANDARD || sc == DMA
 }
 
 // UnmarshalText unmarshals storage class from its textual form into
@@ -103,6 +122,14 @@ func (sc *StorageClass) UnmarshalText(b []byte) error {
 	if scStr == "" {
 		return nil
 	}
+	if scStr == DMAWrite {
+		sc.DMA = DMAWrite
+		return nil
+	}
+	if scStr == DMAReadWrite {
+		sc.DMA = DMAReadWrite
+		return nil
+	}
 	s, err := parseStorageClass(scStr)
 	if err != nil {
 		return err
@@ -116,14 +143,14 @@ func (sc *StorageClass) MarshalText() ([]byte, error) {
 	if sc.Parity != 0 {
 		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
 	}
-	return []byte(""), nil
+	return []byte(sc.DMA), nil
 }
 
 func (sc *StorageClass) String() string {
 	if sc.Parity != 0 {
 		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
 	}
-	return ""
+	return sc.DMA
 }
 
 // Parses given storageClassEnv and returns a storageClass structure.
@@ -212,6 +239,11 @@ func (sCfg Config) GetParityForSC(sc string) (parity int) {
 	}
 }
 
+// GetDMA - returns DMA configuration.
+func (sCfg Config) GetDMA() string {
+	return sCfg.DMA.DMA
+}
+
 // Enabled returns if etcd is enabled.
 func Enabled(kvs config.KVS) bool {
 	ssc := kvs.Get(ClassStandard)
@@ -231,6 +263,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 
 	ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
 	rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS))
+	dma := env.Get(DMAEnv, kvs.Get(ClassDMA))
 	// Check for environment variables and parse into storageClass struct
 	if ssc != "" {
 		cfg.Standard, err = parseStorageClass(ssc)
@@ -252,6 +285,14 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 		cfg.RRS.Parity = defaultRRSParity
 	}
 
+	if dma == "" {
+		dma = defaultDMA
+	}
+	if dma != DMAReadWrite && dma != DMAWrite {
+		return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
+	}
+	cfg.DMA.DMA = dma
+
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
 	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {

cmd/consolelogger.go

@@ -40,8 +40,8 @@ type HTTPConsoleLoggerSys struct {
 	logBuf   *ring.Ring
 }
 
-func mustGetNodeName(endpointZones EndpointZones) (nodeName string) {
+func mustGetNodeName(endpointServerPools EndpointServerPools) (nodeName string) {
-	host, err := xnet.ParseHost(GetLocalPeer(endpointZones))
+	host, err := xnet.ParseHost(GetLocalPeer(endpointServerPools))
 	if err != nil {
 		logger.FatalIf(err, "Unable to start console logging subsystem")
 	}
@@ -63,8 +63,8 @@ func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
 }
 
 // SetNodeName - sets the node name if any after distributed setup has initialized
-func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointZones EndpointZones) {
+func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerPools EndpointServerPools) {
-	sys.nodeName = mustGetNodeName(endpointZones)
+	sys.nodeName = mustGetNodeName(endpointServerPools)
 }
 
 // HasLogListeners returns true if console log listeners are registered

cmd/crypto/header.go

@@ -19,231 +19,62 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"net/http"
-	"strings"
 
-	jsoniter "github.com/json-iterator/go"
 	xhttp "github.com/minio/minio/cmd/http"
 )
 
-// SSEHeader is the general AWS SSE HTTP header key.
-const SSEHeader = "X-Amz-Server-Side-Encryption"
-
-const (
-	// SSEKmsID is the HTTP header key referencing the SSE-KMS
-	// key ID.
-	SSEKmsID = SSEHeader + "-Aws-Kms-Key-Id"
-
-	// SSEKmsContext is the HTTP header key referencing the
-	// SSE-KMS encryption context.
-	SSEKmsContext = SSEHeader + "-Context"
-)
-
-const (
-	// SSECAlgorithm is the HTTP header key referencing
-	// the SSE-C algorithm.
-	SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
-
-	// SSECKey is the HTTP header key referencing the
-	// SSE-C client-provided key..
-	SSECKey = SSEHeader + "-Customer-Key"
-
-	// SSECKeyMD5 is the HTTP header key referencing
-	// the MD5 sum of the client-provided key.
-	SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
-)
-
-const (
-	// SSECopyAlgorithm is the HTTP header key referencing
-	// the SSE-C algorithm for SSE-C copy requests.
-	SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
-
-	// SSECopyKey is the HTTP header key referencing the SSE-C
-	// client-provided key for SSE-C copy requests.
-	SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
-
-	// SSECopyKeyMD5 is the HTTP header key referencing the
-	// MD5 sum of the client key for SSE-C copy requests.
-	SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
-)
-
-const (
-	// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
-	// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
-	// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
-	SSEAlgorithmAES256 = "AES256"
-
-	// SSEAlgorithmKMS is the value of 'X-Amz-Server-Side-Encryption' for SSE-KMS.
-	// See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
-	SSEAlgorithmKMS = "aws:kms"
-)
-
 // RemoveSensitiveHeaders removes confidential encryption
 // information - e.g. the SSE-C key - from the HTTP headers.
 // It has the same semantics as RemoveSensitiveEntires.
 func RemoveSensitiveHeaders(h http.Header) {
-	h.Del(SSECKey)
+	h.Del(xhttp.AmzServerSideEncryptionCustomerKey)
-	h.Del(SSECopyKey)
+	h.Del(xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	h.Del(xhttp.AmzMetaUnencryptedContentLength)
 	h.Del(xhttp.AmzMetaUnencryptedContentMD5)
 }
 
-// IsRequested returns true if the HTTP headers indicates
-// that any form server-side encryption (SSE-C, SSE-S3 or SSE-KMS)
-// is requested.
-func IsRequested(h http.Header) bool {
-	return S3.IsRequested(h) || SSEC.IsRequested(h) || SSECopy.IsRequested(h) || S3KMS.IsRequested(h)
-}
-
-// S3 represents AWS SSE-S3. It provides functionality to handle
-// SSE-S3 requests.
-var S3 = s3{}
-
-type s3 struct{}
-
-// IsRequested returns true if the HTTP headers indicates that
-// the S3 client requests SSE-S3.
-func (s3) IsRequested(h http.Header) bool {
-	_, ok := h[SSEHeader]
-	return ok && strings.ToLower(h.Get(SSEHeader)) != SSEAlgorithmKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
-}
-
-// ParseHTTP parses the SSE-S3 related HTTP headers and checks
-// whether they contain valid values.
-func (s3) ParseHTTP(h http.Header) (err error) {
-	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
-		err = ErrInvalidEncryptionMethod
-	}
-	return
-}
-
-// S3KMS represents AWS SSE-KMS. It provides functionality to
-// handle SSE-KMS requests.
-var S3KMS = s3KMS{}
-
-type s3KMS struct{}
-
-// IsRequested returns true if the HTTP headers indicates that
-// the S3 client requests SSE-KMS.
-func (s3KMS) IsRequested(h http.Header) bool {
-	if _, ok := h[SSEKmsID]; ok {
-		return true
-	}
-	if _, ok := h[SSEKmsContext]; ok {
-		return true
-	}
-	if _, ok := h[SSEHeader]; ok {
-		return strings.ToUpper(h.Get(SSEHeader)) != SSEAlgorithmAES256 // Return only true if the SSE header is specified and does not contain the SSE-S3 value
-	}
-	return false
-}
-
-// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
-// and context, if present, on success.
-func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
-	algorithm := h.Get(SSEHeader)
-	if algorithm != SSEAlgorithmKMS {
-		return "", nil, ErrInvalidEncryptionMethod
-	}
-
-	contextStr, ok := h[SSEKmsContext]
-	if ok {
-		var context map[string]interface{}
-		var json = jsoniter.ConfigCompatibleWithStandardLibrary
-		if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
-			return "", nil, err
-		}
-		return h.Get(SSEKmsID), context, nil
-	}
-	return h.Get(SSEKmsID), nil, nil
-}
-
 var (
-	// SSEC represents AWS SSE-C. It provides functionality to handle
-	// SSE-C requests.
-	SSEC = ssec{}
-
 	// SSECopy represents AWS SSE-C for copy requests. It provides
 	// functionality to handle SSE-C copy requests.
 	SSECopy = ssecCopy{}
 )
 
-type ssec struct{}
 type ssecCopy struct{}
 
-// IsRequested returns true if the HTTP headers contains
-// at least one SSE-C header. SSE-C copy headers are ignored.
-func (ssec) IsRequested(h http.Header) bool {
-	if _, ok := h[SSECAlgorithm]; ok {
-		return true
-	}
-	if _, ok := h[SSECKey]; ok {
-		return true
-	}
-	if _, ok := h[SSECKeyMD5]; ok {
-		return true
-	}
-	return false
-}
-
 // IsRequested returns true if the HTTP headers contains
 // at least one SSE-C copy header. Regular SSE-C headers
 // are ignored.
 func (ssecCopy) IsRequested(h http.Header) bool {
-	if _, ok := h[SSECopyAlgorithm]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm]; ok {
 		return true
 	}
-	if _, ok := h[SSECopyKey]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKey]; ok {
 		return true
 	}
-	if _, ok := h[SSECopyKeyMD5]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5]; ok {
 		return true
 	}
 	return false
 }
 
-// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
-// on success. SSE-C copy headers are ignored.
-func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
-		return key, ErrInvalidCustomerAlgorithm
-	}
-	if h.Get(SSECKey) == "" {
-		return key, ErrMissingCustomerKey
-	}
-	if h.Get(SSECKeyMD5) == "" {
-		return key, ErrMissingCustomerKeyMD5
-	}
-
-	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
-	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
-		return key, ErrInvalidCustomerKey
-	}
-	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
-	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
-		return key, ErrCustomerKeyMD5Mismatch
-	}
-	copy(key[:], clientKey)
-	return key, nil
-}
-
 // ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
 // on success. Regular SSE-C headers are ignored.
 func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != xhttp.AmzEncryptionAES {
 		return key, ErrInvalidCustomerAlgorithm
 	}
-	if h.Get(SSECopyKey) == "" {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey) == "" {
 		return key, ErrMissingCustomerKey
 	}
-	if h.Get(SSECopyKeyMD5) == "" {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5) == "" {
 		return key, ErrMissingCustomerKeyMD5
 	}
 
-	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey))
 	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
 		return key, ErrInvalidCustomerKey
 	}
-	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5))
 	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
 		return key, ErrCustomerKeyMD5Mismatch
 	}

cmd/crypto/header_test.go

@@ -18,26 +18,29 @@ import (
 	"net/http"
 	"sort"
 	"testing"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 func TestIsRequested(t *testing.T) {
 	for i, test := range kmsIsRequestedTests {
-		if got := IsRequested(test.Header) && S3KMS.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && S3KMS.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 	for i, test := range s3IsRequestedTests {
-		if got := IsRequested(test.Header) && S3.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && S3.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 	for i, test := range ssecIsRequestedTests {
-		if got := IsRequested(test.Header) && SSEC.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
-			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		got = got && SSEC.IsRequested(test.Header)
-		}
+		if got != test.Expected {
-	}
-	for i, test := range ssecCopyIsRequestedTests {
-		if got := IsRequested(test.Header) && SSECopy.IsRequested(test.Header); got != test.Expected {
 			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
@@ -131,11 +134,11 @@ var s3IsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool
 }{
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},                // 0
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},        // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},               // 1
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},               // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},                      // 2
-	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},         // 3
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},                // 3
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{SSEAlgorithmKMS}}, Expected: false}, // 4
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{xhttp.AmzEncryptionKMS}}, Expected: false}, // 4
 }
 
 func TestS3IsRequested(t *testing.T) {
@@ -403,7 +406,7 @@ func TestSSECopyParse(t *testing.T) {
 		if err == nil && key == zeroKey {
 			t.Errorf("Test %d: parsed client key is zero key", i)
 		}
-		if _, ok := test.Header[SSECKey]; ok {
+		if _, ok := test.Header[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
 			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
 		}
 	}
@@ -414,47 +417,47 @@ var removeSensitiveHeadersTests = []struct {
 }{
 	{
 		Header: http.Header{
-			SSECKey:    []string{""},
+			xhttp.AmzServerSideEncryptionCustomerKey:     []string{""},
-			SSECopyKey: []string{""},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{""},
 		},
 		ExpectedHeader: http.Header{},
 	},
 	{ // Standard SSE-C request headers
 		Header: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
-			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 	},
 	{ // Standard SSE-C + SSE-C-copy request headers
 		Header: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm:  []string{xhttp.AmzEncryptionAES},
-			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKey:        []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:     []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm:  []string{xhttp.AmzEncryptionAES},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:     []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 	},
 	{ // Standard SSE-C + metadata request headers
 		Header: http.Header{
-			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
-			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			"X-Amz-Meta-Test-1": []string{"Test-1"},
+			"X-Amz-Meta-Test-1":                            []string{"Test-1"},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
-			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			"X-Amz-Meta-Test-1": []string{"Test-1"},
+			"X-Amz-Meta-Test-1":                            []string{"Test-1"},
 		},
 	},
 	{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w

cmd/crypto/kes.go

@@ -177,11 +177,10 @@ func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateK
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var context bytes.Buffer
+	context := ctx.AppendTo(make([]byte, 0, 128))
-	ctx.WriteTo(&context)
 
 	var plainKey []byte
-	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context)
 	if err != nil {
 		return key, nil, err
 	}
@@ -200,11 +199,10 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var context bytes.Buffer
+	context := ctx.AppendTo(make([]byte, 0, 128))
-	ctx.WriteTo(&context)
 
 	var plainKey []byte
-	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context.Bytes())
+	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context)
 	if err != nil {
 		return key, err
 	}
@@ -415,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
 		}
 
 		// If the error is not temp. / retryable => fail the request immediately.
-		if !xnet.IsNetworkOrHostDown(err) &&
+		if !xnet.IsNetworkOrHostDown(err, false) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {

cmd/crypto/key.go

@@ -103,7 +103,6 @@ func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) Se
 func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
 	var (
 		unsealConfig sio.Config
-		decryptedKey bytes.Buffer
 	)
 	switch sealedKey.Algorithm {
 	default:
@@ -122,10 +121,9 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
 	}
 
-	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
+	if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil {
 		return ErrSecretKeyMismatch
 	}
-	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -165,11 +163,7 @@ func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
 	if !IsETagSealed(etag) {
 		return etag, nil
 	}
-	var buffer bytes.Buffer
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write([]byte("SSE-etag"))
-	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+	return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)})
-		return nil, err
-	}
-	return buffer.Bytes(), nil
 }

cmd/crypto/kms.go

@@ -39,6 +39,8 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	sortedKeys := make(sort.StringSlice, 0, len(c))
 	for k := range c {
@@ -67,6 +69,53 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	return n + int64(nn), err
 }
 
+// AppendTo appends the context in a canonical from to dst.
+//
+// AppendTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
+func (c Context) AppendTo(dst []byte) (output []byte) {
+	if len(c) == 0 {
+		return append(dst, '{', '}')
+	}
+
+	// out should not escape.
+	out := bytes.NewBuffer(dst)
+
+	// No need to copy+sort
+	if len(c) == 1 {
+		for k, v := range c {
+			out.WriteString(`{"`)
+			out.WriteString(k)
+			out.WriteString(`":"`)
+			out.WriteString(v)
+			out.WriteString(`"}`)
+		}
+		return out.Bytes()
+	}
+
+	sortedKeys := make([]string, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	out.WriteByte('{')
+	for i, k := range sortedKeys {
+		out.WriteByte('"')
+		out.WriteString(k)
+		out.WriteString(`":"`)
+		out.WriteString(c[k])
+		out.WriteByte('"')
+		if i < len(sortedKeys)-1 {
+			out.WriteByte(',')
+		}
+	}
+	out.WriteByte('}')
+	return out.Bytes()
+}
+
 // KMS represents an active and authenticted connection
 // to a Key-Management-Service. It supports generating
 // data key generation and unsealing of KMS-generated
@@ -155,13 +204,12 @@ func (kms *masterKeyKMS) Info() (info KMSInfo) {
 
 func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
 	var (
-		buffer     bytes.Buffer
 		derivedKey = kms.deriveKey(keyID, ctx)
 	)
-	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+	out, err := sio.DecryptBuffer(key[:0], sealedKey, sio.Config{Key: derivedKey[:]})
+	if err != nil || len(out) != 32 {
 		return key, err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], buffer.Bytes())
 	return key, nil
 }
 
@@ -171,7 +219,7 @@ func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte)
 	}
 	mac := hmac.New(sha256.New, kms.masterKey[:])
 	mac.Write([]byte(keyID))
-	context.WriteTo(mac)
+	mac.Write(context.AppendTo(make([]byte, 0, 128)))
 	mac.Sum(key[:0])
 	return key
 }

cmd/crypto/kms_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"bytes"
+	"fmt"
 	"path"
 	"strings"
 	"testing"
@@ -83,3 +84,32 @@ func TestContextWriteTo(t *testing.T) {
 		}
 	}
 }
+
+func TestContextAppendTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		dst := make([]byte, 0, 1024)
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+		// Append one more
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON+test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON+test.ExpectedJSON)
+		}
+	}
+}
+
+func BenchmarkContext_AppendTo(b *testing.B) {
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	for _, test := range tests {
+		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
+			dst := make([]byte, 0, 1024)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				dst = test.AppendTo(dst[:0])
+			}
+		})
+	}
+}

cmd/crypto/metadata.go

@@ -15,19 +15,42 @@
 package crypto
 
 import (
-	"context"
-	"encoding/base64"
-	"errors"
-
 	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
+)
+
+const (
+	// MetaMultipart indicates that the object has been uploaded
+	// in multiple parts - via the S3 multipart API.
+	MetaMultipart = "X-Minio-Internal-Encrypted-Multipart"
+
+	// MetaIV is the random initialization vector (IV) used for
+	// the MinIO-internal key derivation.
+	MetaIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
+
+	// MetaAlgorithm is the algorithm used to derive internal keys
+	// and encrypt the objects.
+	MetaAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
+
+	// MetaSealedKeySSEC is the sealed object encryption key in case of SSE-C.
+	MetaSealedKeySSEC = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
+	// MetaSealedKeyS3 is the sealed object encryption key in case of SSE-S3
+	MetaSealedKeyS3 = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// MetaSealedKeyKMS is the sealed object encryption key in case of SSE-KMS
+	MetaSealedKeyKMS = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"
+
+	// MetaKeyID is the KMS master key ID used to generate/encrypt the data
+	// encryption key (DEK).
+	MetaKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// MetaDataEncryptionKey is the sealed data encryption key (DEK) received from
+	// the KMS.
+	MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
 )
 
 // IsMultiPart returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption
 // and the S3 multipart API.
 func IsMultiPart(metadata map[string]string) bool {
-	if _, ok := metadata[SSEMultipart]; ok {
+	if _, ok := metadata[MetaMultipart]; ok {
 		return true
 	}
 	return false
@@ -37,8 +60,8 @@ func IsMultiPart(metadata map[string]string) bool {
 // information - e.g. the SSE-C key - from the metadata map.
 // It has the same semantics as RemoveSensitiveHeaders.
 func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
-	delete(metadata, SSECKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
-	delete(metadata, SSECopyKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
 	delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
 }
@@ -46,31 +69,36 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 // RemoveSSEHeaders removes all crypto-specific SSE
 // header entries from the metadata map.
 func RemoveSSEHeaders(metadata map[string]string) {
-	delete(metadata, SSEHeader)
+	delete(metadata, xhttp.AmzServerSideEncryption)
-	delete(metadata, SSEKmsID)
+	delete(metadata, xhttp.AmzServerSideEncryptionKmsID)
-	delete(metadata, SSEKmsContext)
+	delete(metadata, xhttp.AmzServerSideEncryptionKmsContext)
-	delete(metadata, SSECKeyMD5)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerAlgorithm)
-	delete(metadata, SSECAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKeyMD5)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5)
 }
 
 // RemoveInternalEntries removes all crypto-specific internal
 // metadata entries from the metadata map.
 func RemoveInternalEntries(metadata map[string]string) {
-	delete(metadata, SSEMultipart)
+	delete(metadata, MetaMultipart)
-	delete(metadata, SSEIV)
+	delete(metadata, MetaAlgorithm)
-	delete(metadata, SSESealAlgorithm)
+	delete(metadata, MetaIV)
-	delete(metadata, SSECSealedKey)
+	delete(metadata, MetaSealedKeySSEC)
-	delete(metadata, S3SealedKey)
+	delete(metadata, MetaSealedKeyS3)
-	delete(metadata, S3KMSKeyID)
+	delete(metadata, MetaSealedKeyKMS)
-	delete(metadata, S3KMSSealedKey)
+	delete(metadata, MetaKeyID)
+	delete(metadata, MetaDataEncryptionKey)
 }
 
 // IsSourceEncrypted returns true if the source is encrypted
 func IsSourceEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSECAlgorithm]; ok {
+	if _, ok := metadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
 		return true
 	}
-	if _, ok := metadata[SSEHeader]; ok {
+	if _, ok := metadata[xhttp.AmzServerSideEncryption]; ok {
 		return true
 	}
 	return false
@@ -82,10 +110,10 @@ func IsSourceEncrypted(metadata map[string]string) bool {
 // IsEncrypted only checks whether the metadata contains at least
 // one entry indicating SSE-C or SSE-S3.
 func IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSEIV]; ok {
+	if _, ok := metadata[MetaIV]; ok {
 		return true
 	}
-	if _, ok := metadata[SSESealAlgorithm]; ok {
+	if _, ok := metadata[MetaAlgorithm]; ok {
 		return true
 	}
 	if IsMultiPart(metadata) {
@@ -97,28 +125,7 @@ func IsEncrypted(metadata map[string]string) bool {
 	if SSEC.IsEncrypted(metadata) {
 		return true
 	}
-	return false
+	if S3KMS.IsEncrypted(metadata) {
-}
-
-// IsEncrypted returns true if the object metadata indicates
-// that the object was uploaded using SSE-S3.
-func (s3) IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[S3SealedKey]; ok {
-		return true
-	}
-	if _, ok := metadata[S3KMSKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[S3KMSSealedKey]; ok {
-		return true
-	}
-	return false
-}
-
-// IsEncrypted returns true if the object metadata indicates
-// that the object was uploaded using SSE-C.
-func (ssec) IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSECSealedKey]; ok {
 		return true
 	}
 	return false
@@ -129,157 +136,11 @@ func (ssec) IsEncrypted(metadata map[string]string) bool {
 // metadata is nil.
 func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	if metadata == nil {
-		return map[string]string{SSEMultipart: ""}
+		return map[string]string{MetaMultipart: ""}
 	}
-	metadata[SSEMultipart] = ""
+	metadata[MetaMultipart] = ""
 	return metadata
 }
 
-// CreateMetadata encodes the sealed object key into the metadata and returns
-// the modified metadata. If the keyID and the kmsKey is not empty it encodes
-// both into the metadata as well. It allocates a new metadata map if metadata
-// is nil.
-func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
-	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
-	}
-
-	// There are two possibilites:
-	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
-	// - We use a K/V -> There must be no key ID and no KMS data key.
-	// Otherwise, the caller has passed an invalid argument combination.
-	if keyID == "" && len(kmsKey) != 0 {
-		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
-	}
-	if keyID != "" && len(kmsKey) == 0 {
-		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
-	}
-
-	if metadata == nil {
-		metadata = make(map[string]string, 5)
-	}
-
-	metadata[SSESealAlgorithm] = sealedKey.Algorithm
-	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
-	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
-		metadata[S3KMSKeyID] = keyID
-		metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
-	}
-	return metadata
-}
-
-// ParseMetadata extracts all SSE-S3 related values from the object metadata
-// and checks whether they are well-formed. It returns the sealed object key
-// on success. If the metadata contains both, a KMS master key ID and a sealed
-// KMS data key it returns both. If the metadata does not contain neither a
-// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
-// KMS data key. Otherwise, it returns an error.
-func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
-	// Extract all required values from object metadata
-	b64IV, ok := metadata[SSEIV]
-	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalIV
-	}
-	algorithm, ok := metadata[SSESealAlgorithm]
-	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
-	}
-	b64SealedKey, ok := metadata[S3SealedKey]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
-	}
-
-	// There are two possibilites:
-	// - We use a KMS -> There must be a key ID and a KMS data key.
-	// - We use a K/V -> There must be no key ID and no KMS data key.
-	// Otherwise, the metadata is corrupted.
-	keyID, idPresent := metadata[S3KMSKeyID]
-	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
-	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
-	}
-	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
-	}
-
-	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
-		return keyID, kmsKey, sealedKey, errInvalidInternalIV
-	}
-	if algorithm != SealAlgorithm {
-		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
-	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
-	}
-	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
-		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
-		if err != nil {
-			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
-		}
-	}
-
-	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
-	return keyID, kmsKey, sealedKey, nil
-}
-
-// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
-// It allocates a new metadata map if metadata is nil.
-func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
-	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
-	}
-
-	if metadata == nil {
-		metadata = make(map[string]string, 3)
-	}
-	metadata[SSESealAlgorithm] = SealAlgorithm
-	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
-	metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	return metadata
-}
-
-// ParseMetadata extracts all SSE-C related values from the object metadata
-// and checks whether they are well-formed. It returns the sealed object key
-// on success.
-func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
-	// Extract all required values from object metadata
-	b64IV, ok := metadata[SSEIV]
-	if !ok {
-		return sealedKey, errMissingInternalIV
-	}
-	algorithm, ok := metadata[SSESealAlgorithm]
-	if !ok {
-		return sealedKey, errMissingInternalSealAlgorithm
-	}
-	b64SealedKey, ok := metadata[SSECSealedKey]
-	if !ok {
-		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
-	}
-
-	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
-		return sealedKey, errInvalidInternalIV
-	}
-	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
-		return sealedKey, errInvalidInternalSealAlgorithm
-	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
-	}
-
-	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
-	return sealedKey, nil
-}
-
 // IsETagSealed returns true if the etag seems to be encrypted.
 func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -27,9 +27,9 @@ var isMultipartTests = []struct {
 	Metadata  map[string]string
 	Multipart bool
 }{
-	{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}},                           // 0
+	{Multipart: true, Metadata: map[string]string{MetaMultipart: ""}},                          // 0
 	{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
-	{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}},                 // 2
+	{Multipart: true, Metadata: map[string]string{MetaMultipart: "some-value"}},                // 2
 	{Multipart: false, Metadata: map[string]string{"": ""}},                                    // 3
 	{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
 }
@@ -46,13 +46,13 @@ var isEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}},                               // 0
+	{Encrypted: true, Metadata: map[string]string{MetaMultipart: ""}},                              // 0
-	{Encrypted: true, Metadata: map[string]string{SSEIV: ""}},                                      // 1
+	{Encrypted: true, Metadata: map[string]string{MetaIV: ""}},                                     // 1
-	{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}},                           // 2
+	{Encrypted: true, Metadata: map[string]string{MetaAlgorithm: ""}},                              // 2
-	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                          // 3
-	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
-	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
-	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -69,13 +69,13 @@ var s3IsEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}},                             // 0
-	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{MetaIV: ""}},                                    // 1
-	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
-	{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}},                             // 3
+	{Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                         // 3
-	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
-	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
+	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
-	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -92,13 +92,13 @@ var ssecIsEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
+	{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}},                             // 0
-	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
+	{Encrypted: false, Metadata: map[string]string{MetaIV: ""}},                                    // 1
-	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
+	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
-	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                          // 3
-	{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}},                               // 4
+	{Encrypted: false, Metadata: map[string]string{MetaSealedKeyS3: ""}},                           // 4
-	{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}},                                // 5
+	{Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}},                                 // 5
-	{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}},                            // 6
+	{Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                     // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -121,65 +121,65 @@ var s3ParseMetadataTests = []struct {
 }{
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
 	{
-		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
+		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaDataEncryptionKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
 	{
 		ExpectedErr: errInvalidInternalIV,
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 5
 	{
 		ExpectedErr: errInvalidInternalSealAlgorithm,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
 		ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeyS3: "",
-			S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
 		ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "key-1",
-			S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
+			MetaDataEncryptionKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
 		},
 		DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
 	}, // 8
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
 	}, // 9
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
+			MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), MetaKeyID: "key-1",
-			S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
+			MetaDataEncryptionKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
 		},
 		DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
 	}, // 10
@@ -223,43 +223,43 @@ var ssecParseMetadataTests = []struct {
 
 	SealedKey SealedKey
 }{
-	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                      // 0
-	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
+	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
 		ExpectedErr: errInvalidInternalIV,
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeySSEC: ""}, SealedKey: SealedKey{},
 	}, // 3
 	{
 		ExpectedErr: errInvalidInternalSealAlgorithm,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeySSEC: "",
 		},
 		SealedKey: SealedKey{},
 	}, // 4
 	{
 		ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeySSEC: "",
 		},
 		SealedKey: SealedKey{},
 	}, // 5
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
-			SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(make([]byte, 64)),
 		},
 		SealedKey: SealedKey{Algorithm: SealAlgorithm},
 	}, // 6
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
+			MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: InsecureSealAlgorithm,
-			SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
+			MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
 		},
 		SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
 	}, // 7
@@ -267,8 +267,8 @@ var ssecParseMetadataTests = []struct {
 
 func TestCreateMultipartMetadata(t *testing.T) {
 	metadata := CreateMultipartMetadata(nil)
-	if v, ok := metadata[SSEMultipart]; !ok || v != "" {
+	if v, ok := metadata[MetaMultipart]; !ok || v != "" {
-		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
+		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", MetaMultipart, v, "")
 	}
 }
 
@@ -411,20 +411,20 @@ var removeInternalEntriesTests = []struct {
 }{
 	{ // 0
 		Metadata: map[string]string{
-			SSEMultipart:     "",
+			MetaMultipart:         "",
-			SSEIV:            "",
+			MetaIV:                "",
-			SSESealAlgorithm: "",
+			MetaAlgorithm:         "",
-			SSECSealedKey:    "",
+			MetaSealedKeySSEC:     "",
-			S3SealedKey:      "",
+			MetaSealedKeyS3:       "",
-			S3KMSKeyID:       "",
+			MetaKeyID:             "",
-			S3KMSSealedKey:   "",
+			MetaDataEncryptionKey: "",
 		},
 		Expected: map[string]string{},
 	},
 	{ // 1
 		Metadata: map[string]string{
-			SSEMultipart:         "",
+			MetaMultipart:        "",
-			SSEIV:                "",
+			MetaIV:               "",
 			"X-Amz-Meta-A":       "X",
 			"X-Minio-Internal-B": "Y",
 		},

cmd/crypto/sse-c.go

@@ -0,0 +1,157 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"net/http"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type ssec struct{}
+
+var (
+	// SSEC represents AWS SSE-C. It provides functionality to handle
+	// SSE-C requests.
+	SSEC = ssec{}
+
+	_ Type = SSEC
+)
+
+// String returns the SSE domain as string. For SSE-C the
+// domain is "SSE-C".
+func (ssec) String() string { return "SSE-C" }
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the metadata contains an SSE-C
+// entry inidicating that the object has been encrypted using
+// SSE-C.
+func (ssec) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeySSEC]; ok {
+		return true
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm) != xhttp.AmzEncryptionAES {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (s3 ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := s3.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// CreateMetadata encodes the sealed key into the metadata
+// and returns the modified metadata. It allocates a new
+// metadata map if metadata is nil.
+func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 3)
+	}
+	metadata[MetaAlgorithm] = SealAlgorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeySSEC] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-C related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success.
+func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeySSEC]
+	if !ok {
+		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
+		return sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return sealedKey, nil
+}

cmd/crypto/sse-kms.go

@@ -0,0 +1,201 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"path"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type ssekms struct{}
+
+var (
+	// S3KMS represents AWS SSE-KMS. It provides functionality to
+	// handle SSE-KMS requests.
+	S3KMS = ssekms{}
+
+	_ Type = S3KMS
+)
+
+// String returns the SSE domain as string. For SSE-KMS the
+// domain is "SSE-KMS".
+func (ssekms) String() string { return "SSE-KMS" }
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-KMS header.
+func (ssekms) IsRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
+		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
+// and the KMS context on success.
+func (ssekms) ParseHTTP(h http.Header) (string, Context, error) {
+	algorithm := h.Get(xhttp.AmzServerSideEncryption)
+	if algorithm != xhttp.AmzEncryptionKMS {
+		return "", nil, ErrInvalidEncryptionMethod
+	}
+
+	var ctx Context
+	if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		if err := json.Unmarshal([]byte(context[0]), &ctx); err != nil {
+			return "", nil, err
+		}
+	}
+	return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-KMS.
+func (ssekms) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeyKMS]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return true
+	}
+	return false
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (s3 ssekms) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	if err != nil {
+		return key, err
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return key, err
+	}
+	err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
+	return key, err
+}
+
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
+func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 5)
+	}
+
+	metadata[MetaAlgorithm] = sealedKey.Algorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[MetaKeyID] = keyID
+		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-KMS related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
+func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[MetaKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+	}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+	}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}

cmd/crypto/sse-s3.go

@@ -0,0 +1,181 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"path"
+	"strings"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type sses3 struct{}
+
+var (
+	// S3 represents AWS SSE-S3. It provides functionality to handle
+	// SSE-S3 requests.
+	S3 = sses3{}
+
+	_ Type = S3
+)
+
+// String returns the SSE domain as string. For SSE-S3 the
+// domain is "SSE-S3".
+func (sses3) String() string { return "SSE-S3" }
+
+func (sses3) IsRequested(h http.Header) bool {
+	_, ok := h[xhttp.AmzServerSideEncryption]
+	return ok && strings.ToLower(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
+}
+
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (sses3) ParseHTTP(h http.Header) error {
+	if h.Get(xhttp.AmzServerSideEncryption) != xhttp.AmzEncryptionAES {
+		return ErrInvalidEncryptionMethod
+	}
+	return nil
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-S3.
+func (sses3) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeyS3]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return true
+	}
+	return false
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (s3 sses3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	if err != nil {
+		return key, err
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return key, err
+	}
+	err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
+	return key, err
+}
+
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
+func (sses3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 5)
+	}
+
+	metadata[MetaAlgorithm] = sealedKey.Algorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeyS3] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[MetaKeyID] = keyID
+		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-S3 related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
+func (sses3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeyS3]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[MetaKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+	}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+	}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}

cmd/crypto/sse.go

@@ -17,44 +17,15 @@ package crypto
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
-	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/sio"
 )
 
-const (
-	// SSEMultipart is the metadata key indicating that the object
-	// was uploaded using the S3 multipart API and stored using
-	// some from of server-side-encryption.
-	SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
-
-	// SSEIV is the metadata key referencing the random initialization
-	// vector (IV) used for SSE-S3 and SSE-C key derivation.
-	SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
-
-	// SSESealAlgorithm is the metadata key referencing the algorithm
-	// used by SSE-C and SSE-S3 to encrypt the object.
-	SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
-
-	// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
-	SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
-
-	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
-	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
-
-	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
-	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
-	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
-
-	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
-	// by KMS. It is only used for SSE-S3 + KMS.
-	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
-)
-
 const (
 	// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
 	// the key-encryption-key and to en/decrypt the object data.
@@ -67,39 +38,34 @@ const (
 	InsecureSealAlgorithm = "DARE-SHA256"
 )
 
-// String returns the SSE domain as string. For SSE-S3 the
+// Type represents an AWS SSE type:
-// domain is "SSE-S3".
+//  • SSE-C
-func (s3) String() string { return "SSE-S3" }
+//  • SSE-S3
+//  • SSE-KMS
+type Type interface {
+	fmt.Stringer
 
-// UnsealObjectKey extracts and decrypts the sealed object key
+	IsRequested(http.Header) bool
-// from the metadata using KMS and returns the decrypted object
+
-// key.
+	IsEncrypted(map[string]string) bool
-func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
-	if err != nil {
-		return
-	}
-	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
-	if err != nil {
-		return
-	}
-	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
-	return
 }
 
-// String returns the SSE domain as string. For SSE-C the
+// IsRequested returns true and the SSE Type if the HTTP headers
-// domain is "SSE-C".
+// indicate that some form server-side encryption is requested.
-func (ssec) String() string { return "SSE-C" }
+//
-
+// If no SSE headers are present then IsRequested returns false
-// UnsealObjectKey extracts and decrypts the sealed object key
+// and no Type.
-// from the metadata using the SSE-C client key of the HTTP headers
+func IsRequested(h http.Header) (Type, bool) {
-// and returns the decrypted object key.
+	switch {
-func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	case S3.IsRequested(h):
-	clientKey, err := sse.ParseHTTP(h)
+		return S3, true
-	if err != nil {
+	case S3KMS.IsRequested(h):
-		return
+		return S3KMS, true
+	case SSEC.IsRequested(h):
+		return SSEC, true
+	default:
+		return nil, false
 	}
-	return unsealObjectKey(clientKey, metadata, bucket, object)
 }
 
 // UnsealObjectKey extracts and decrypts the sealed object key

cmd/crypto/vault.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -224,11 +223,10 @@ func (v *vaultService) CreateKey(keyID string) error {
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var contextStream bytes.Buffer
+	context := ctx.AppendTo(make([]byte, 0, 128))
-	ctx.WriteTo(&contextStream)
 
 	payload := map[string]interface{}{
-		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context": base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
@@ -260,12 +258,11 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var contextStream bytes.Buffer
+	context := ctx.AppendTo(make([]byte, 0, 128))
-	ctx.WriteTo(&contextStream)
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
@@ -294,12 +291,11 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) {
-	var contextStream bytes.Buffer
+	context := ctx.AppendTo(make([]byte, 0, 128))
-	ctx.WriteTo(&contextStream)
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {

cmd/data-crawler.go

@@ -21,19 +21,21 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"math"
 	"math/rand"
 	"os"
 	"path"
-	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/crawler"
+	"github.com/minio/minio/cmd/config/heal"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
@@ -43,7 +45,6 @@ import (
 
 const (
 	dataCrawlSleepPerFolder  = time.Millisecond // Time to wait between folders.
-	dataCrawlSleepDefMult    = 10.0             // Default multiplier for waits between operations.
 	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
 	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
 
@@ -53,8 +54,12 @@ const (
 )
 
 var (
-	globalCrawlerConfig          crawler.Config
+	globalHealConfig   heal.Config
-	dataCrawlerLeaderLockTimeout = newDynamicTimeout(1*time.Minute, 30*time.Second)
+	globalHealConfigMu sync.Mutex
+
+	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	// Sleeper values are updated when config is loaded.
+	crawlerSleeper = newDynamicSleeper(10, 10*time.Second)
 )
 
 // initDataCrawler will start the crawler unless disabled.
@@ -69,10 +74,10 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
 		if err != nil {
 			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
 			continue
@@ -95,11 +100,21 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
+	crawlTimer := time.NewTimer(dataCrawlStartDelay)
+	defer crawlTimer.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.NewTimer(dataCrawlStartDelay).C:
+		case <-crawlTimer.C:
+			// Reset the timer for next cycle.
+			crawlTimer.Reset(dataCrawlStartDelay)
+
+			if intDataUpdateTracker.debug {
+				console.Debugln("starting crawler cycle")
+			}
+
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
@@ -135,27 +150,26 @@ type cachedFolder struct {
 }
 
 type folderScanner struct {
-	root               string
+	root       string
-	getSize            getSizeFn
+	getSize    getSizeFn
-	oldCache           dataUsageCache
+	oldCache   dataUsageCache
-	newCache           dataUsageCache
+	newCache   dataUsageCache
-	withFilter         *bloomFilter
+	withFilter *bloomFilter
-	waitForLowActiveIO func()
 
-	dataUsageCrawlMult  float64
 	dataUsageCrawlDebug bool
 	healFolderInclude   uint32 // Include a clean folder one in n cycles.
 	healObjectSelect    uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 
 	newFolders      []cachedFolder
 	existingFolders []cachedFolder
+	disks           []StorageAPI
 }
 
 // crawlDataFolder will crawl the basepath+cache.Info.Name and return an updated cache.
 // The returned cache will always be valid, but may not be updated from the existing.
-// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// Before each operation sleepDuration is called which can be used to temporarily halt the crawler.
 // If the supplied context is canceled the function will return at the first chance.
-func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
@@ -172,26 +186,30 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		return cache, errors.New("internal error: root scan attempted")
 	}
 
-	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		delayMult = dataCrawlSleepDefMult
-	}
-
 	s := folderScanner{
 		root:                basePath,
 		getSize:             getSize,
 		oldCache:            cache,
 		newCache:            dataUsageCache{Info: cache.Info},
-		waitForLowActiveIO:  waitForLowActiveIO,
 		newFolders:          nil,
 		existingFolders:     nil,
-		dataUsageCrawlMult:  delayMult,
 		dataUsageCrawlDebug: intDataUpdateTracker.debug,
 		healFolderInclude:   0,
 		healObjectSelect:    0,
 	}
 
+	// Add disks for set healing.
+	if len(cache.Disks) > 0 {
+		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
+		if ok {
+			s.disks = objAPI.GetDisksID(cache.Disks...)
+			if len(s.disks) != len(cache.Disks) {
+				logger.Info(logPrefix+"Missing disks, want %d, found %d. Cannot heal."+logSuffix, len(cache.Disks), len(s.disks))
+				s.disks = s.disks[:0]
+			}
+		}
+	}
+
 	// Enable healing in XL mode.
 	if globalIsErasure {
 		// Include a clean folder one in n cycles.
@@ -347,16 +365,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 
 		// If there are lifecycle rules for the prefix, remove the filter.
 		filter := f.withFilter
+		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil {
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
-			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
+			if f.dataUsageCrawlDebug {
-			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+				logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				}
-				activeLifeCycle = f.oldCache.Info.lifeCycle
-				filter = nil
 			}
+			activeLifeCycle = f.oldCache.Info.lifeCycle
+			filter = nil
 		}
 		if _, ok := f.oldCache.Cache[thisHash.Key()]; filter != nil && ok {
 			// If folder isn't in filter and we have data, skip it completely.
@@ -376,8 +392,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				}
 			}
 		}
-		f.waitForLowActiveIO()
+		crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
-		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
 
 		cache := dataUsageEntry{}
 
@@ -424,9 +439,9 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				}
 				return nil
 			}
-			f.waitForLowActiveIO()
+
 			// Dynamic time delay.
-			t := UTCNow()
+			wait := crawlerSleeper.Timer(ctx)
 
 			// Get file size, ignore errors.
 			item := crawlItem{
@@ -437,18 +452,18 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
-				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
+				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
-			size, err := f.getSize(item)
+			sizeSummary, err := f.getSize(item)
 
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			wait()
 			if err == errSkipFile {
 				return nil
 			}
 			logger.LogIf(ctx, err)
-			cache.Size += size
+			cache.addSizes(sizeSummary)
 			cache.Objects++
-			cache.ObjSizes.add(size)
+			cache.ObjSizes.add(sizeSummary.totalSize)
 
 			return nil
 		})
@@ -462,8 +477,8 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			continue
 		}
 
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
-		if objAPI == nil {
+		if !ok || len(f.disks) == 0 {
 			continue
 		}
 
@@ -483,28 +498,137 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		// We therefore perform a heal check.
 		// If that doesn't bring it back we remove the folder and assume it was deleted.
 		// This means that the next run will not look for it.
+		// How to resolve results.
+		resolver := metadataResolutionParams{
+			dirQuorum: getReadQuorum(len(f.disks)),
+			objQuorum: getReadQuorum(len(f.disks)),
+			bucket:    "",
+		}
+
 		for k := range existing {
-			f.waitForLowActiveIO()
 			bucket, prefix := path2BucketObject(k)
 			if f.dataUsageCrawlDebug {
 				logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
 			}
 
-			err = objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: healDeleteDangling},
+			// Dynamic time delay.
-				func(bucket, object, versionID string) error {
+			wait := crawlerSleeper.Timer(ctx)
-					return bgSeq.queueHealTask(healSource{
+			resolver.bucket = bucket
-						bucket:    bucket,
-						object:    object,
-						versionID: versionID,
-					}, madmin.HealItemObject)
-				})
 
-			if f.dataUsageCrawlDebug && err != nil {
+			foundObjs := false
-				logger.Info(color.Green("healObjects:")+" checking returned value %v", err)
+			dangling := true
+			ctx, cancel := context.WithCancel(ctx)
+			err := listPathRaw(ctx, listPathRawOptions{
+				disks:          f.disks,
+				bucket:         bucket,
+				path:           prefix,
+				recursive:      true,
+				reportNotFound: true,
+				minDisks:       len(f.disks), // We want full consistency.
+				// Weird, maybe transient error.
+				agreed: func(entry metaCacheEntry) {
+					if f.dataUsageCrawlDebug {
+						logger.Info(color.Green("healObjects:")+" got agreement: %v", entry.name)
+					}
+					if entry.isObject() {
+						dangling = false
+					}
+				},
+				// Some disks have data for this.
+				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+					if f.dataUsageCrawlDebug {
+						logger.Info(color.Green("healObjects:")+" got partial, %d agreed, errs: %v", nAgreed, errs)
+					}
+					// Sleep and reset.
+					wait()
+					wait = crawlerSleeper.Timer(ctx)
+					entry, ok := entries.resolve(&resolver)
+					if !ok {
+						for _, err := range errs {
+							if err != nil {
+								// Not all disks are ready, do nothing for now.
+								dangling = false
+								return
+							}
+						}
+
+						// If no errors, queue it for healing.
+						entry, _ = entries.firstFound()
+					}
+
+					if f.dataUsageCrawlDebug {
+						logger.Info(color.Green("healObjects:")+" resolved to: %v, dir: %v", entry.name, entry.isDir())
+					}
+					if entry.isDir() {
+						return
+					}
+					dangling = false
+					// We got an entry which we should be able to heal.
+					fiv, err := entry.fileInfoVersions(bucket)
+					if err != nil {
+						err := bgSeq.queueHealTask(healSource{
+							bucket:    bucket,
+							object:    entry.name,
+							versionID: "",
+						}, madmin.HealItemObject)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
+						foundObjs = foundObjs || err == nil
+						return
+					}
+					for _, ver := range fiv.Versions {
+						// Sleep and reset.
+						wait()
+						wait = crawlerSleeper.Timer(ctx)
+						err := bgSeq.queueHealTask(healSource{
+							bucket:    bucket,
+							object:    fiv.Name,
+							versionID: ver.VersionID,
+						}, madmin.HealItemObject)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
+						foundObjs = foundObjs || err == nil
+					}
+				},
+				// Too many disks failed.
+				finished: func(errs []error) {
+					if f.dataUsageCrawlDebug {
+						logger.Info(color.Green("healObjects:")+" too many errors: %v", errs)
+					}
+					dangling = false
+					cancel()
+				},
+			})
+
+			if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound {
+				logger.Info(color.Green("healObjects:")+" checking returned value %v (%T)", err, err)
 			}
 
+			// If we found one or more disks with this folder, delete it.
+			if err == nil && dangling {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("healObjects:")+" deleting dangling directory %s", prefix)
+				}
+				// If we have quorum, found directories, but no objects, issue heal to delete the dangling.
+				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: true},
+					func(bucket, object, versionID string) error {
+						// Wait for each heal as per crawler frequency.
+						wait()
+						wait = crawlerSleeper.Timer(ctx)
+						return bgSeq.queueHealTask(healSource{
+							bucket:    bucket,
+							object:    object,
+							versionID: versionID,
+						}, madmin.HealItemObject)
+					})
+			}
+
+			wait()
+
 			// Add unless healing returned an error.
-			if err == nil {
+			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
 				cache.addChild(hashPath(k))
 				if final {
@@ -535,17 +659,16 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 		default:
 		}
 
-		f.waitForLowActiveIO()
 		if typ&os.ModeDir != 0 {
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
 			dirStack = dirStack[:len(dirStack)-1]
-			sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+			crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 			return err
 		}
 
 		// Dynamic time delay.
-		t := UTCNow()
+		wait := crawlerSleeper.Timer(ctx)
 
 		// Get file size, ignore errors.
 		dirStack = append(dirStack, entName)
@@ -554,16 +677,14 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 
 		bucket, prefix := path2BucketObjectWithBasePath(f.root, fileName)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil {
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
-			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
+			if f.dataUsageCrawlDebug {
-				if f.dataUsageCrawlDebug {
+				logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				}
-				activeLifeCycle = f.oldCache.Info.lifeCycle
 			}
+			activeLifeCycle = f.oldCache.Info.lifeCycle
 		}
 
-		size, err := f.getSize(
+		sizeSummary, err := f.getSize(
 			crawlItem{
 				Path:       fileName,
 				Typ:        typ,
@@ -572,19 +693,19 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
-				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
+				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			})
 
-		// Don't sleep for really small amount of time
+		// Wait to throttle IO
-		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+		wait()
 
 		if err == errSkipFile {
 			return nil
 		}
 		logger.LogIf(ctx, err)
-		cache.Size += size
+		cache.addSizes(sizeSummary)
 		cache.Objects++
-		cache.ObjSizes.add(size)
+		cache.ObjSizes.add(sizeSummary.totalSize)
 		return nil
 	}
 	err := readDirFn(path.Join(dirStack...), addDir)
@@ -607,7 +728,15 @@ type crawlItem struct {
 	debug      bool
 }
 
-type getSizeFn func(item crawlItem) (int64, error)
+type sizeSummary struct {
+	totalSize      int64
+	replicatedSize int64
+	pendingSize    int64
+	failedSize     int64
+	replicaSize    int64
+}
+
+type getSizeFn func(item crawlItem) (sizeSummary, error)
 
 // transformMetaDir will transform a directory to prefix/file.ext
 func (i *crawlItem) transformMetaDir() {
@@ -639,7 +768,11 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	}
 	if i.heal {
 		if i.debug {
-			logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID)
+			if meta.oi.VersionID != "" {
+				logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID)
+			} else {
+				logger.Info(color.Green("applyActions:")+" heal checking: %v/%v", i.bucket, i.objectPath())
+			}
 		}
 		res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling})
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -652,6 +785,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		size = res.ObjectSize
 	}
 	if i.lifeCycle == nil {
+		if i.debug {
+			logger.Info(color.Green("applyActions:")+" no lifecycle rules to apply: %q", i.objectPath())
+		}
 		return size
 	}
 
@@ -666,14 +802,26 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			IsLatest:         meta.oi.IsLatest,
 			NumVersions:      meta.numVersions,
 			SuccessorModTime: meta.successorModTime,
+			RestoreOngoing:   meta.oi.RestoreOngoing,
+			RestoreExpires:   meta.oi.RestoreExpires,
+			TransitionStatus: meta.oi.TransitionStatus,
 		})
 	if i.debug {
-		logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
+		if versionID != "" {
+			logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
+		} else {
+			logger.Info(color.Green("applyActions:")+" lifecycle: %q Initial scan: %v", i.objectPath(), action)
+		}
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 	default:
 		// No action.
+		if i.debug {
+			logger.Info(color.Green("applyActions:")+" object not expirable: %q", i.objectPath())
+		}
 		return size
 	}
 
@@ -700,22 +848,27 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	size = obj.Size
 
 	// Recalculate action.
-	action = i.lifeCycle.ComputeAction(
+	lcOpts := lifecycle.ObjectOpts{
-		lifecycle.ObjectOpts{
+		Name:             i.objectPath(),
-			Name:             i.objectPath(),
+		UserTags:         obj.UserTags,
-			UserTags:         obj.UserTags,
+		ModTime:          obj.ModTime,
-			ModTime:          obj.ModTime,
+		VersionID:        obj.VersionID,
-			VersionID:        obj.VersionID,
+		DeleteMarker:     obj.DeleteMarker,
-			DeleteMarker:     obj.DeleteMarker,
+		IsLatest:         obj.IsLatest,
-			IsLatest:         obj.IsLatest,
+		NumVersions:      meta.numVersions,
-			NumVersions:      meta.numVersions,
+		SuccessorModTime: meta.successorModTime,
-			SuccessorModTime: meta.successorModTime,
+		RestoreOngoing:   obj.RestoreOngoing,
-		})
+		RestoreExpires:   obj.RestoreExpires,
+		TransitionStatus: obj.TransitionStatus,
+	}
+	action = i.lifeCycle.ComputeAction(lcOpts)
 	if i.debug {
 		logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 	default:
 		// No action.
 		return size
@@ -723,7 +876,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 
 	opts := ObjectOptions{}
 	switch action {
-	case lifecycle.DeleteVersionAction:
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
 		if obj.VersionID == "" {
 			return size
@@ -732,14 +885,40 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			locked := enforceRetentionForDeletion(ctx, obj)
 			if locked {
 				if i.debug {
-					logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath())
+					if obj.VersionID != "" {
+						logger.Info(color.Green("applyActions:")+" lifecycle: %s v%s is locked, not deleting", i.objectPath(), obj.VersionID)
+					} else {
+						logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath())
+					}
 				}
 				return size
 			}
 		}
 		opts.VersionID = obj.VersionID
-	case lifecycle.DeleteAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction:
 		opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+		if obj.TransitionStatus == "" {
+			opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
+			opts.VersionID = obj.VersionID
+			opts.TransitionStatus = lifecycle.TransitionPending
+			if _, err = o.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
+				// Assume it is still there.
+				logger.LogIf(ctx, err)
+				return size
+			}
+		}
+		globalTransitionState.queueTransitionTask(obj)
+		return 0
+	}
+
+	if obj.TransitionStatus != "" {
+		if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), lcOpts, action, false); err != nil {
+			logger.LogIf(ctx, err)
+			return size
+		}
+		// Notification already sent at *deleteTransitionedObject*, return '0' here.
+		return 0
 	}
 
 	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
@@ -749,9 +928,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		return size
 	}
 
+	eventName := event.ObjectRemovedDelete
+	if obj.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+
 	// Notify object deleted event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectRemovedDelete,
+		EventName:  eventName,
 		BucketName: i.bucket,
 		Object:     obj,
 		Host:       "Internal: [ILM-EXPIRY]",
@@ -764,22 +948,176 @@ func (i *crawlItem) objectPath() string {
 	return path.Join(i.prefix, i.objectName)
 }
 
-// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// healReplication will heal a scanned item that has failed replication.
-// sleep is limited to max 1 second.
+func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta, sizeS *sizeSummary) {
-func sleepDuration(d time.Duration, x float64) {
+	if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
-	// Don't sleep for really small amount of time
+		// heal delete marker replication failure or versioned delete replication failure
-	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if meta.oi.ReplicationStatus == replication.Pending ||
-		if d > time.Second {
+			meta.oi.ReplicationStatus == replication.Failed ||
-			d = time.Second
+			meta.oi.VersionPurgeStatus == Failed || meta.oi.VersionPurgeStatus == Pending {
+			i.healReplicationDeletes(ctx, o, meta)
+			return
 		}
-		time.Sleep(d)
+	}
+	switch meta.oi.ReplicationStatus {
+	case replication.Pending:
+		sizeS.pendingSize += meta.oi.Size
+		globalReplicationState.queueReplicaTask(meta.oi)
+	case replication.Failed:
+		sizeS.failedSize += meta.oi.Size
+		globalReplicationState.queueReplicaTask(meta.oi)
+	case replication.Complete:
+		sizeS.replicatedSize += meta.oi.Size
+	case replication.Replica:
+		sizeS.replicaSize += meta.oi.Size
 	}
 }
 
-// healReplication will heal a scanned item that has failed replication.
+// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta) {
+func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, meta actionMeta) {
-	if meta.oi.ReplicationStatus == replication.Pending ||
+	// handle soft delete and permanent delete failures here.
-		meta.oi.ReplicationStatus == replication.Failed {
+	if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
-		globalReplicationState.queueReplicaTask(meta.oi)
+		versionID := ""
+		dmVersionID := ""
+		if meta.oi.VersionPurgeStatus.Empty() {
+			dmVersionID = meta.oi.VersionID
+		} else {
+			versionID = meta.oi.VersionID
+		}
+		globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+			DeletedObject: DeletedObject{
+				ObjectName:                    meta.oi.Name,
+				DeleteMarkerVersionID:         dmVersionID,
+				VersionID:                     versionID,
+				DeleteMarkerReplicationStatus: string(meta.oi.ReplicationStatus),
+				DeleteMarkerMTime:             DeleteMarkerMTime{meta.oi.ModTime},
+				DeleteMarker:                  meta.oi.DeleteMarker,
+				VersionPurgeStatus:            meta.oi.VersionPurgeStatus,
+			},
+			Bucket: meta.oi.Bucket,
+		})
 	}
 }
+
+type dynamicSleeper struct {
+	mu sync.RWMutex
+
+	// Sleep factor
+	factor float64
+
+	// maximum sleep cap,
+	// set to <= 0 to disable.
+	maxSleep time.Duration
+
+	// Don't sleep at all, if time taken is below this value.
+	// This is to avoid too small costly sleeps.
+	minSleep time.Duration
+
+	// cycle will be closed
+	cycle chan struct{}
+}
+
+// newDynamicSleeper
+func newDynamicSleeper(factor float64, maxWait time.Duration) *dynamicSleeper {
+	return &dynamicSleeper{
+		factor:   factor,
+		cycle:    make(chan struct{}),
+		maxSleep: maxWait,
+		minSleep: 100 * time.Microsecond,
+	}
+}
+
+// Timer returns a timer that has started.
+// When the returned function is called it will wait.
+func (d *dynamicSleeper) Timer(ctx context.Context) func() {
+	t := time.Now()
+	return func() {
+		doneAt := time.Now()
+		for {
+			// Grab current values
+			d.mu.RLock()
+			minWait, maxWait := d.minSleep, d.maxSleep
+			factor := d.factor
+			cycle := d.cycle
+			d.mu.RUnlock()
+			elapsed := doneAt.Sub(t)
+			// Don't sleep for really small amount of time
+			wantSleep := time.Duration(float64(elapsed) * factor)
+			if wantSleep <= minWait {
+				return
+			}
+			if maxWait > 0 && wantSleep > maxWait {
+				wantSleep = maxWait
+			}
+			timer := time.NewTimer(wantSleep)
+			select {
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			case <-timer.C:
+				return
+			case <-cycle:
+				if !timer.Stop() {
+					// We expired.
+					<-timer.C
+					return
+				}
+			}
+		}
+	}
+}
+
+// Sleep sleeps the specified time multiplied by the sleep factor.
+// If the factor is updated the sleep will be done again with the new factor.
+func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
+	for {
+		// Grab current values
+		d.mu.RLock()
+		minWait, maxWait := d.minSleep, d.maxSleep
+		factor := d.factor
+		cycle := d.cycle
+		d.mu.RUnlock()
+		// Don't sleep for really small amount of time
+		wantSleep := time.Duration(float64(base) * factor)
+		if wantSleep <= minWait {
+			return
+		}
+		if maxWait > 0 && wantSleep > maxWait {
+			wantSleep = maxWait
+		}
+		timer := time.NewTimer(wantSleep)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return
+		case <-timer.C:
+			return
+		case <-cycle:
+			if !timer.Stop() {
+				// We expired.
+				<-timer.C
+				return
+			}
+		}
+	}
+}
+
+// Update the current settings and cycle all waiting.
+// Parameters are the same as in the contructor.
+func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if math.Abs(d.factor-factor) < 1e-10 && d.maxSleep == maxWait {
+		return nil
+	}
+	// Update values and cycle waiting.
+	close(d.cycle)
+	d.factor = factor
+	d.maxSleep = maxWait
+	d.cycle = make(chan struct{})
+	return nil
+}

cmd/data-update-tracker.go

@@ -40,13 +40,13 @@ import (
 
 const (
 	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 1000000
+	dataUpdateTrackerEstItems = 10000000
 	// ... we want this false positive rate:
 	dataUpdateTrackerFP        = 0.99
-	dataUpdateTrackerQueueSize = 10000
+	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 3
+	dataUpdateTrackerVersion      = 4
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -79,7 +79,7 @@ func newDataUpdateTracker() *dataUpdateTracker {
 		Current: dataUpdateFilter{
 			idx: 1,
 		},
-		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn,
+		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn || serverDebugLog,
 		input:      make(chan string, dataUpdateTrackerQueueSize),
 		save:       make(chan struct{}, 1),
 		saveExited: make(chan struct{}),
@@ -168,6 +168,43 @@ func (d *dataUpdateTracker) current() uint64 {
 	return d.Current.idx
 }
 
+// latestWithDir returns the highest index that contains the directory.
+// This means that any cycle higher than this does NOT contain the entry.
+func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
+	bucket, _ := path2BucketObjectWithBasePath("", dir)
+	if bucket == "" {
+		if d.debug && len(dir) > 0 {
+			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", dir)
+		}
+		return d.current()
+	}
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, dir)
+		}
+		return d.current()
+	}
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
+		return d.Current.idx
+	}
+	if d.debug {
+		logger.Info("current bloom does NOT contains dir %s", dir)
+	}
+
+	idx := d.Current.idx - 1
+	for {
+		f := d.History.find(idx)
+		if f == nil || f.bf.containsDir(dir) || idx == 0 {
+			break
+		}
+		idx--
+	}
+	return idx
+}
+
 // start will load the current data from the drives start collecting information and
 // start a saver goroutine.
 // All of these will exit when the context is canceled.
@@ -198,7 +235,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
 		f, err := os.Open(cacheFormatPath)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if osIsNotExist(err) {
 				continue
 			}
 			logger.LogIf(ctx, err)
@@ -263,7 +300,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
 			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
 			err := ioutil.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
 			if err != nil {
-				if os.IsNotExist(err) {
+				if osIsNotExist(err) {
 					continue
 				}
 				logger.LogIf(ctx, err)
@@ -366,7 +403,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2:
+	case 1, 2, 3:
 		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		return nil
 	case dataUpdateTrackerVersion:
@@ -440,41 +477,66 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
+	for in := range d.input {
-		select {
+		bucket, _ := path2BucketObjectWithBasePath("", in)
-		case <-ctx.Done():
+		if bucket == "" {
-			return
+			if d.debug && len(in) > 0 {
-		case in := <-d.input:
+				logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s)", in)
-				}
-				continue
 			}
-
+			continue
-			if isReservedOrInvalidBucket(bucket, false) {
-				if false && d.debug {
-					logger.Info(color.Green("data-usage:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
-				}
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until level 3.
-			d.mu.Lock()
-			for i := range split {
-				if d.debug && false {
-					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
-				}
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
 		}
+
+		if isReservedOrInvalidBucket(bucket, false) {
+			if d.debug {
+				logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+			}
+			continue
+		}
+		split := splitPathDeterministic(in)
+
+		// Add all paths until done.
+		d.mu.Lock()
+		for i := range split {
+			if d.debug {
+				logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+			}
+			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+		}
+		d.dirty = d.dirty || len(split) > 0
+		d.mu.Unlock()
 	}
 }
 
+// markDirty adds the supplied path to the current bloom filter.
+func (d *dataUpdateTracker) markDirty(in string) {
+	bucket, _ := path2BucketObjectWithBasePath("", in)
+	if bucket == "" {
+		if d.debug && len(in) > 0 {
+			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
+		}
+		return
+	}
+
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug && false {
+			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+		}
+		return
+	}
+	split := splitPathDeterministic(in)
+
+	// Add all paths until done.
+	d.mu.Lock()
+	for i := range split {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+		}
+		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+	}
+	d.dirty = d.dirty || len(split) > 0
+	d.mu.Unlock()
+}
+
 // find entry with specified index.
 // Returns nil if not found.
 func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
@@ -534,8 +596,13 @@ func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint6
 // cycleFilter will cycle the bloom filter to start recording to index y if not already.
 // The response will contain a bloom filter starting at index x up to, but not including index y.
 // If y is 0, the response will not update y, but return the currently recorded information
-// from the up until and including current y.
+// from the oldest (unless 0, then it will be all) until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
+	if req.OldestClean != "" {
+		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
+	}
+	current := req.Current
+	oldest := req.Oldest
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	if current == 0 {
@@ -543,7 +610,10 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
 		}
 		d.History.sort()
-		return d.filterFrom(ctx, d.History[len(d.History)-1].idx, d.Current.idx), nil
+		if oldest == 0 {
+			oldest = d.History[len(d.History)-1].idx
+		}
+		return d.filterFrom(ctx, oldest, d.Current.idx), nil
 	}
 
 	// Move current to history if new one requested
@@ -571,7 +641,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 // Trailing slashes are removed.
 // Returns 0 length if no parts are found after trimming.
 func splitPathDeterministic(in string) []string {
-	split := strings.Split(in, SlashSeparator)
+	split := strings.Split(decodeDirObject(in), SlashSeparator)
 
 	// Trim empty start/end
 	for len(split) > 0 {
@@ -587,10 +657,6 @@ func splitPathDeterministic(in string) []string {
 		split = split[:len(split)-1]
 	}
 
-	// Return up to 3 parts.
-	if len(split) > 3 {
-		split = split[:3]
-	}
 	return split
 }
 
@@ -599,6 +665,9 @@ func splitPathDeterministic(in string) []string {
 type bloomFilterRequest struct {
 	Oldest  uint64
 	Current uint64
+	// If set the oldest clean version will be returned in OldestIdx
+	// and the rest of the request will be ignored.
+	OldestClean string
 }
 
 type bloomFilterResponse struct {
@@ -615,10 +684,9 @@ type bloomFilterResponse struct {
 }
 
 // ObjectPathUpdated indicates a path has been updated.
-// The function will never block.
+// The function will block until the entry has been picked up.
 func ObjectPathUpdated(s string) {
-	select {
+	if intDataUpdateTracker != nil {
-	case objectUpdatedCh <- s:
+		intDataUpdateTracker.markDirty(s)
-	default:
 	}
 }

cmd/data-update-tracker_test.go

@@ -169,7 +169,12 @@ func TestDataUpdateTracker(t *testing.T) {
 		})
 	}
 	// Cycle to history
-	_, err = dut.cycleFilter(ctx, 1, 2)
+	req := bloomFilterRequest{
+		Oldest:  1,
+		Current: 2,
+	}
+
+	_, err = dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -200,7 +205,11 @@ func TestDataUpdateTracker(t *testing.T) {
 	if dut.current() != 2 {
 		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
 	}
-	bfr2, err := dut.cycleFilter(ctx, 1, 3)
+	req = bloomFilterRequest{
+		Oldest:  1,
+		Current: 3,
+	}
+	bfr2, err := dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/data-usage-cache.go

@@ -46,16 +46,20 @@ type sizeHistogram [dataUsageBucketLen]uint64
 //msgp:tuple dataUsageEntry
 type dataUsageEntry struct {
 	// These fields do no include any children.
-	Size     int64
+	Size                   int64
-	Objects  uint64
+	ReplicatedSize         uint64
-	ObjSizes sizeHistogram
+	ReplicationPendingSize uint64
-
+	ReplicationFailedSize  uint64
-	Children dataUsageHashMap
+	ReplicaSize            uint64
+	Objects                uint64
+	ObjSizes               sizeHistogram
+	Children               dataUsageHashMap
 }
 
 // dataUsageCache contains a cache of data usage entries.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
+	Disks []string
 	Cache map[string]dataUsageEntry
 }
 
@@ -75,10 +79,23 @@ type dataUsageCacheInfo struct {
 	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }
 
+func (e *dataUsageEntry) addSizes(summary sizeSummary) {
+	e.Size += summary.totalSize
+	e.ReplicatedSize += uint64(summary.replicatedSize)
+	e.ReplicationFailedSize += uint64(summary.failedSize)
+	e.ReplicationPendingSize += uint64(summary.pendingSize)
+	e.ReplicaSize += uint64(summary.replicaSize)
+}
+
 // merge other data usage entry into this, excluding children.
 func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Size += other.Size
+	e.ReplicationPendingSize += other.ReplicationPendingSize
+	e.ReplicationFailedSize += other.ReplicationFailedSize
+	e.ReplicatedSize += other.ReplicatedSize
+	e.ReplicaSize += other.ReplicaSize
+
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
 	}
@@ -212,11 +229,15 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	}
 	flat := d.flatten(*e)
 	return DataUsageInfo{
-		LastUpdate:        d.Info.LastUpdate,
+		LastUpdate:             d.Info.LastUpdate,
-		ObjectsTotalCount: flat.Objects,
+		ObjectsTotalCount:      flat.Objects,
-		ObjectsTotalSize:  uint64(flat.Size),
+		ObjectsTotalSize:       uint64(flat.Size),
-		BucketsCount:      uint64(len(e.Children)),
+		ReplicatedSize:         flat.ReplicatedSize,
-		BucketsUsage:      d.bucketsUsageInfo(buckets),
+		ReplicationFailedSize:  flat.ReplicationFailedSize,
+		ReplicationPendingSize: flat.ReplicationPendingSize,
+		ReplicaSize:            flat.ReplicaSize,
+		BucketsCount:           uint64(len(e.Children)),
+		BucketsUsage:           d.bucketsUsageInfo(buckets),
 	}
 }
 
@@ -342,9 +363,13 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		}
 		flat := d.flatten(*e)
 		dst[bucket.Name] = BucketUsageInfo{
-			Size:                 uint64(flat.Size),
+			Size:                   uint64(flat.Size),
-			ObjectsCount:         flat.Objects,
+			ObjectsCount:           flat.Objects,
-			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+			ReplicationPendingSize: flat.ReplicationPendingSize,
+			ReplicatedSize:         flat.ReplicatedSize,
+			ReplicationFailedSize:  flat.ReplicationFailedSize,
+			ReplicaSize:            flat.ReplicaSize,
+			ObjectSizesHistogram:   flat.ObjSizes.toMap(),
 		}
 	}
 	return dst
@@ -359,9 +384,13 @@ func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
 	}
 	flat := d.flatten(*e)
 	return BucketUsageInfo{
-		Size:                 uint64(flat.Size),
+		Size:                   uint64(flat.Size),
-		ObjectsCount:         flat.Objects,
+		ObjectsCount:           flat.Objects,
-		ObjectSizesHistogram: flat.ObjSizes.toMap(),
+		ReplicationPendingSize: flat.ReplicationPendingSize,
+		ReplicatedSize:         flat.ReplicatedSize,
+		ReplicationFailedSize:  flat.ReplicationFailedSize,
+		ReplicaSize:            flat.ReplicaSize,
+		ObjectSizesHistogram:   flat.ObjSizes.toMap(),
 	}
 }
 
@@ -440,7 +469,11 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) && !errors.Is(err, InsufficientReadQuorum{}) {
+		switch err.(type) {
+		case ObjectNotFound:
+		case BucketNotFound:
+		case InsufficientReadQuorum:
+		default:
 			return toObjectErr(err, dataUsageBucket, name)
 		}
 		*d = dataUsageCache{}
@@ -456,9 +489,12 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
-	b := d.serialize()
+	pr, pw := io.Pipe()
-	size := int64(len(b))
+	go func() {
-	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+		pw.CloseWithError(d.serializeTo(pw))
+	}()
+	defer pr.Close()
+	r, err := hash.NewReader(pr, -1, "", "", -1, false)
 	if err != nil {
 		return err
 	}
@@ -477,35 +513,36 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // dataUsageCacheVer indicates the cache version.
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
-const dataUsageCacheVer = 2
+const dataUsageCacheVer = 3
 
 // serialize the contents of the cache.
-func (d *dataUsageCache) serialize() []byte {
+func (d *dataUsageCache) serializeTo(dst io.Writer) error {
-	// Prepend version and compress.
+	// Add version and compress.
-	dst := make([]byte, 0, d.Msgsize()+1)
+	_, err := dst.Write([]byte{dataUsageCacheVer})
-	dst = append(dst, dataUsageCacheVer)
+	if err != nil {
-	buf := bytes.NewBuffer(dst)
+		return err
-	enc, err := zstd.NewWriter(buf,
+	}
+	enc, err := zstd.NewWriter(dst,
 		zstd.WithEncoderLevel(zstd.SpeedFastest),
 		zstd.WithWindowSize(1<<20),
 		zstd.WithEncoderConcurrency(2))
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		return err
-		return nil
 	}
 	mEnc := msgp.NewWriter(enc)
 	err = d.EncodeMsg(mEnc)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		return err
-		return nil
+	}
+	err = mEnc.Flush()
+	if err != nil {
+		return err
 	}
-	mEnc.Flush()
 	err = enc.Close()
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		return err
-		return nil
 	}
-	return buf.Bytes()
+	return nil
 }
 
 // deserialize the supplied byte slice into the cache.
@@ -516,7 +553,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		return io.ErrUnexpectedEOF
 	}
 	switch b[0] {
-	case 1:
+	case 1, 2:
 		return errors.New("cache version deprecated (will autoupdate)")
 	case dataUsageCacheVer:
 	default:

cmd/data-usage-cache_gen.go

@@ -30,35 +30,54 @@ func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Cache":
+		case "Disks":
 			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntry, zb0002)
+				z.Cache = make(map[string]dataUsageEntry, zb0003)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0002 > 0 {
+			for zb0003 > 0 {
-				zb0002--
+				zb0003--
-				var za0001 string
+				var za0002 string
-				var za0002 dataUsageEntry
+				var za0003 dataUsageEntry
-				za0001, err = dc.ReadString()
+				za0002, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0002.DecodeMsg(dc)
+				err = za0003.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
+					err = msgp.WrapError(err, "Cache", za0002)
 					return
 				}
-				z.Cache[za0001] = za0002
+				z.Cache[za0002] = za0003
 			}
 		default:
 			err = dc.Skip()
@@ -73,9 +92,9 @@ func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
+	// map header, size 3
 	// write "Info"
-	err = en.Append(0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	err = en.Append(0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
 	if err != nil {
 		return
 	}
@@ -84,6 +103,23 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Info")
 		return
 	}
+	// write "Disks"
+	err = en.Append(0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Disks)))
+	if err != nil {
+		err = msgp.WrapError(err, "Disks")
+		return
+	}
+	for za0001 := range z.Disks {
+		err = en.WriteString(z.Disks[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "Disks", za0001)
+			return
+		}
+	}
 	// write "Cache"
 	err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
 	if err != nil {
@@ -94,15 +130,15 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Cache")
 		return
 	}
-	for za0001, za0002 := range z.Cache {
+	for za0002, za0003 := range z.Cache {
-		err = en.WriteString(za0001)
+		err = en.WriteString(za0002)
 		if err != nil {
 			err = msgp.WrapError(err, "Cache")
 			return
 		}
-		err = za0002.EncodeMsg(en)
+		err = za0003.EncodeMsg(en)
 		if err != nil {
-			err = msgp.WrapError(err, "Cache", za0001)
+			err = msgp.WrapError(err, "Cache", za0002)
 			return
 		}
 	}
@@ -112,22 +148,28 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
+	// map header, size 3
 	// string "Info"
-	o = append(o, 0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o = append(o, 0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
 	o, err = z.Info.MarshalMsg(o)
 	if err != nil {
 		err = msgp.WrapError(err, "Info")
 		return
 	}
+	// string "Disks"
+	o = append(o, 0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Disks)))
+	for za0001 := range z.Disks {
+		o = msgp.AppendString(o, z.Disks[za0001])
+	}
 	// string "Cache"
 	o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
 	o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
-	for za0001, za0002 := range z.Cache {
+	for za0002, za0003 := range z.Cache {
-		o = msgp.AppendString(o, za0001)
+		o = msgp.AppendString(o, za0002)
-		o, err = za0002.MarshalMsg(o)
+		o, err = za0003.MarshalMsg(o)
 		if err != nil {
-			err = msgp.WrapError(err, "Cache", za0001)
+			err = msgp.WrapError(err, "Cache", za0002)
 			return
 		}
 	}
@@ -158,35 +200,54 @@ func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Cache":
+		case "Disks":
 			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntry, zb0002)
+				z.Cache = make(map[string]dataUsageEntry, zb0003)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0002 > 0 {
+			for zb0003 > 0 {
-				var za0001 string
+				var za0002 string
-				var za0002 dataUsageEntry
+				var za0003 dataUsageEntry
-				zb0002--
+				zb0003--
-				za0001, bts, err = msgp.ReadStringBytes(bts)
+				za0002, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0002.UnmarshalMsg(bts)
+				bts, err = za0003.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0001)
+					err = msgp.WrapError(err, "Cache", za0002)
 					return
 				}
-				z.Cache[za0001] = za0002
+				z.Cache[za0002] = za0003
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -202,11 +263,15 @@ func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCache) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
+	for za0001 := range z.Disks {
+		s += msgp.StringPrefixSize + len(z.Disks[za0001])
+	}
+	s += 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0001, za0002 := range z.Cache {
+		for za0002, za0003 := range z.Cache {
-			_ = za0002
+			_ = za0003
-			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
 		}
 	}
 	return
@@ -427,8 +492,8 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 4 {
+	if zb0001 != 8 {
-		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		err = msgp.ArrayError{Wanted: 8, Got: zb0001}
 		return
 	}
 	z.Size, err = dc.ReadInt64()
@@ -436,6 +501,26 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "Size")
 		return
 	}
+	z.ReplicatedSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	z.ReplicationPendingSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	z.ReplicationFailedSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	z.ReplicaSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
 	z.Objects, err = dc.ReadUint64()
 	if err != nil {
 		err = msgp.WrapError(err, "Objects")
@@ -468,8 +553,8 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 4
+	// array header, size 8
-	err = en.Append(0x94)
+	err = en.Append(0x98)
 	if err != nil {
 		return
 	}
@@ -478,6 +563,26 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Size")
 		return
 	}
+	err = en.WriteUint64(z.ReplicatedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicationPendingSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicationFailedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicaSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
 	err = en.WriteUint64(z.Objects)
 	if err != nil {
 		err = msgp.WrapError(err, "Objects")
@@ -506,9 +611,13 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 4
+	// array header, size 8
-	o = append(o, 0x94)
+	o = append(o, 0x98)
 	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.ReplicatedSize)
+	o = msgp.AppendUint64(o, z.ReplicationPendingSize)
+	o = msgp.AppendUint64(o, z.ReplicationFailedSize)
+	o = msgp.AppendUint64(o, z.ReplicaSize)
 	o = msgp.AppendUint64(o, z.Objects)
 	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
 	for za0001 := range z.ObjSizes {
@@ -530,8 +639,8 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 4 {
+	if zb0001 != 8 {
-		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		err = msgp.ArrayError{Wanted: 8, Got: zb0001}
 		return
 	}
 	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
@@ -539,6 +648,26 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "Size")
 		return
 	}
+	z.ReplicatedSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	z.ReplicationPendingSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	z.ReplicationFailedSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	z.ReplicaSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
 	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err, "Objects")
@@ -572,7 +701,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
 	return
 }
 

cmd/data-usage.go

@@ -28,7 +28,6 @@ import (
 
 const (
 	envDataUsageCrawlConf  = "MINIO_DISK_USAGE_CRAWL_ENABLE"
-	envDataUsageCrawlDelay = "MINIO_DISK_USAGE_CRAWL_DELAY"
 	envDataUsageCrawlDebug = "MINIO_DISK_USAGE_CRAWL_DEBUG"
 
 	dataUsageRoot   = SlashSeparator
@@ -40,8 +39,8 @@ const (
 )
 
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
-	for dataUsageInfo := range gui {
+	for dataUsageInfo := range dui {
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
 		if err != nil {
 			logger.LogIf(ctx, err)
@@ -53,7 +52,6 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan
 			logger.LogIf(ctx, err)
 			continue
 		}
-
 		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
 		if !isErrBucketNotFound(err) {
 			logger.LogIf(ctx, err)

cmd/data-usage_test.go

@@ -51,18 +51,20 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item crawlItem) (i int64, err error) {
+	getSize := func(item crawlItem) (sizeS sizeSummary, err error) {
 		if item.Typ&os.ModeDir == 0 {
-			s, err := os.Stat(item.Path)
+			var s os.FileInfo
+			s, err = os.Stat(item.Path)
 			if err != nil {
-				return 0, err
+				return
 			}
-			return s.Size(), nil
+			sizeS.totalSize = s.Size()
+			return sizeS, nil
 		}
-		return 0, nil
+		return
 	}
 
-	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -183,7 +185,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		},
 	}
 	createUsageTestFiles(t, base, bucket, files)
-	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+	got, err = crawlDataFolder(context.Background(), base, got, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -268,7 +270,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -345,17 +347,19 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	createUsageTestFiles(t, base, "", files)
 
-	getSize := func(item crawlItem) (i int64, err error) {
+	getSize := func(item crawlItem) (sizeS sizeSummary, err error) {
 		if item.Typ&os.ModeDir == 0 {
-			s, err := os.Stat(item.Path)
+			var s os.FileInfo
+			s, err = os.Stat(item.Path)
 			if err != nil {
-				return 0, err
+				return
 			}
-			return s.Size(), nil
+			sizeS.totalSize = s.Size()
+			return
 		}
-		return 0, nil
+		return
 	}
-	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -465,7 +469,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		},
 	}
 	createUsageTestFiles(t, base, "", files)
-	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+	got, err = crawlDataFolder(context.Background(), base, got, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -548,7 +552,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -642,28 +646,33 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 	}
 	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item crawlItem) (i int64, err error) {
+	getSize := func(item crawlItem) (sizeS sizeSummary, err error) {
 		if item.Typ&os.ModeDir == 0 {
-			s, err := os.Stat(item.Path)
+			var s os.FileInfo
+			s, err = os.Stat(item.Path)
 			if err != nil {
-				return 0, err
+				return
 			}
-			return s.Size(), nil
+			sizeS.totalSize = s.Size()
+			return
 		}
-		return 0, nil
+		return
 	}
-	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
+	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	var buf bytes.Buffer
-	b := want.serialize()
+	err = want.serializeTo(&buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("serialized size:", buf.Len(), "bytes")
 	var got dataUsageCache
-	err = got.deserialize(bytes.NewBuffer(b))
+	err = got.deserialize(&buf)
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Log("serialized size:", len(b), "bytes")
 	if got.Info.LastUpdate.IsZero() {
 		t.Error("lastupdate not set")
 	}

cmd/disk-cache-backend.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -70,7 +72,9 @@ type cacheMeta struct {
 	// Ranges maps cached range to associated filename.
 	Ranges map[string]string `json:"ranges,omitempty"`
 	// Hits is a counter on the number of times this object has been accessed so far.
-	Hits int `json:"hits,omitempty"`
+	Hits   int    `json:"hits,omitempty"`
+	Bucket string `json:"bucket,omitempty"`
+	Object string `json:"object,omitempty"`
 }
 
 // RangeInfo has the range, file and range length information for a cached range.
@@ -130,19 +134,21 @@ type diskCache struct {
 	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	purgeRunning int32
 
-	triggerGC     chan struct{}
+	triggerGC        chan struct{}
-	dir           string         // caching directory
+	dir              string         // caching directory
-	stats         CacheDiskStats // disk cache stats for prometheus
+	stats            CacheDiskStats // disk cache stats for prometheus
-	quotaPct      int            // max usage in %
+	quotaPct         int            // max usage in %
-	pool          sync.Pool
+	pool             sync.Pool
-	after         int // minimum accesses before an object is cached.
+	after            int // minimum accesses before an object is cached.
-	lowWatermark  int
+	lowWatermark     int
-	highWatermark int
+	highWatermark    int
-	enableRange   bool
+	enableRange      bool
+	commitWriteback  bool
+	retryWritebackCh chan ObjectInfo
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
-	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
+	NewNSLockFn func(cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
@@ -156,15 +162,17 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
-		dir:           dir,
+		dir:              dir,
-		triggerGC:     make(chan struct{}, 1),
+		triggerGC:        make(chan struct{}, 1),
-		stats:         CacheDiskStats{Dir: dir},
+		stats:            CacheDiskStats{Dir: dir},
-		quotaPct:      quotaPct,
+		quotaPct:         quotaPct,
-		after:         config.After,
+		after:            config.After,
-		lowWatermark:  config.WatermarkLow,
+		lowWatermark:     config.WatermarkLow,
-		highWatermark: config.WatermarkHigh,
+		highWatermark:    config.WatermarkHigh,
-		enableRange:   config.Range,
+		enableRange:      config.Range,
-		online:        1,
+		commitWriteback:  config.CommitWriteback,
+		retryWritebackCh: make(chan ObjectInfo, 10000),
+		online:           1,
 		pool: sync.Pool{
 			New: func() interface{} {
 				b := disk.AlignedBlock(int(cacheBlkSize))
@@ -174,9 +182,12 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		nsMutex: newNSLock(false),
 	}
 	go cache.purgeWait(ctx)
+	if cache.commitWriteback {
+		go cache.scanCacheWritebackFailures(ctx)
+	}
 	cache.diskSpaceAvailable(0) // update if cache usage is already high.
-	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
+	cache.NewNSLockFn = func(cachePath string) RWLocker {
-		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
+		return cache.nsMutex.NewNSLock(nil, cachePath, "")
 	}
 	return &cache, nil
 }
@@ -194,9 +205,9 @@ func (c *diskCache) diskUsageLow() bool {
 		logger.LogIf(ctx, err)
 		return false
 	}
-	usedPercent := (di.Used / di.Total) * 100
+	usedPercent := float64(di.Used) * 100 / float64(di.Total)
 	low := int(usedPercent) < gcStopPct
-	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	atomic.StoreUint64(&c.stats.UsagePercent, uint64(usedPercent))
 	if low {
 		atomic.StoreInt32(&c.stats.UsageState, 0)
 	}
@@ -323,6 +334,12 @@ func (c *diskCache) purge(ctx context.Context) {
 		// stat all cached file ranges and cacheDataFile.
 		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, name))
 		objInfo := meta.ToObjectInfo("", "")
+		// prevent gc from clearing un-synced commits. This metadata is present when
+		// cache writeback commit setting is enabled.
+		status, ok := objInfo.UserDefined[writeBackStatusHeader]
+		if ok && status != CommitComplete.String() {
+			return nil
+		}
 		cc := cacheControlOpts(objInfo)
 		for fname, fi := range cachedFiles {
 			if cc != nil {
@@ -419,8 +436,8 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 // if partial object is cached.
 func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	cLock := c.NewNSLockFn(cacheObjPath)
-	if err = cLock.GetRLock(globalOperationTimeout); err != nil {
+	if err = cLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return
 	}
 
@@ -501,8 +518,8 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 // incHitsOnly is true if metadata update is incrementing only the hit counter
 func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachedPath)
+	cLock := c.NewNSLockFn(cachedPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -524,7 +541,11 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	}
 	defer f.Close()
 
-	m := &cacheMeta{Version: cacheMetaVersion}
+	m := &cacheMeta{
+		Version: cacheMetaVersion,
+		Bucket:  bucket,
+		Object:  object,
+	}
 	if err := jsonLoad(f, m); err != nil && err != io.EOF {
 		return err
 	}
@@ -538,12 +559,15 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 			}
 			m.Ranges[rs.String(actualSize)] = rsFileName
 		}
-	} else {
+	}
+	if rs == nil && !incHitsOnly {
 		// this is necessary cleanup of range files if entire object is cached.
-		for _, f := range m.Ranges {
+		if _, err := os.Stat(pathJoin(cachedPath, cacheDataFile)); err == nil {
-			removeAll(pathJoin(cachedPath, f))
+			for _, f := range m.Ranges {
+				removeAll(pathJoin(cachedPath, f))
+			}
+			m.Ranges = nil
 		}
-		m.Ranges = nil
 	}
 	m.Stat.Size = actualSize
 	m.Stat.ModTime = UTCNow()
@@ -558,7 +582,6 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 			m.Meta["etag"] = etag
 		}
 	}
-
 	m.Hits++
 
 	m.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
@@ -570,22 +593,22 @@ func getCacheSHADir(dir, bucket, object string) string {
 }
 
 // Cache data to disk with bitrot checksum added for each block of 1MB
-func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, error) {
+func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, string, error) {
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
-		return 0, err
+		return 0, "", err
 	}
 	filePath := pathJoin(cachePath, fileName)
 
 	if filePath == "" || reader == nil {
-		return 0, errInvalidArgument
+		return 0, "", errInvalidArgument
 	}
 
 	if err := checkPathLength(filePath); err != nil {
-		return 0, err
+		return 0, "", err
 	}
 	f, err := os.Create(filePath)
 	if err != nil {
-		return 0, osErrToFileErr(err)
+		return 0, "", osErrToFileErr(err)
 	}
 	defer f.Close()
 
@@ -595,12 +618,12 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 
 	bufp := c.pool.Get().(*[]byte)
 	defer c.pool.Put(bufp)
-
+	md5Hash := md5.New()
 	var n, n2 int
 	for {
 		n, err = io.ReadFull(reader, *bufp)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			return 0, err
+			return 0, "", err
 		}
 		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && size != 0 {
@@ -609,21 +632,28 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 		}
 		h.Reset()
 		if _, err = h.Write((*bufp)[:n]); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		hashBytes := h.Sum(nil)
+		// compute md5Hash of original data stream if writeback commit to cache
+		if c.commitWriteback {
+			if _, err = md5Hash.Write((*bufp)[:n]); err != nil {
+				return 0, "", err
+			}
+		}
 		if _, err = f.Write(hashBytes); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		if n2, err = f.Write((*bufp)[:n]); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		bytesWritten += int64(n2)
 		if eof {
 			break
 		}
 	}
-	return bytesWritten, nil
+
+	return bytesWritten, base64.StdEncoding.EncodeToString(md5Hash.Sum(nil)), nil
 }
 
 func newCacheEncryptReader(content io.Reader, bucket, object string, metadata map[string]string) (r io.Reader, err error) {
@@ -660,41 +690,41 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 }
 
 // Caches the object to disk
-func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
+func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) (oi ObjectInfo, err error) {
 	if !c.diskSpaceAvailable(size) {
 		io.Copy(ioutil.Discard, data)
-		return errDiskFull
+		return oi, errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachePath)
+	cLock := c.NewNSLockFn(cachePath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
-		return err
+		return oi, err
 	}
 	defer cLock.Unlock()
 
 	meta, _, numHits, err := c.statCache(ctx, cachePath)
 	// Case where object not yet cached
-	if os.IsNotExist(err) && c.after >= 1 {
+	if osIsNotExist(err) && c.after >= 1 {
-		return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+		return oi, c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
 	}
 	// Case where object already has a cache metadata entry but not yet cached
 	if err == nil && numHits < c.after {
 		cETag := extractETag(meta.Meta)
 		bETag := extractETag(opts.UserDefined)
 		if cETag == bETag {
-			return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+			return oi, c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
 		}
 		incHitsOnly = true
 	}
 
 	if rs != nil {
-		return c.putRange(ctx, bucket, object, data, size, rs, opts)
+		return oi, c.putRange(ctx, bucket, object, data, size, rs, opts)
 	}
 	if !c.diskSpaceAvailable(size) {
-		return errDiskFull
+		return oi, errDiskFull
 	}
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
-		return err
+		return oi, err
 	}
 	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
@@ -702,25 +732,39 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
-			return err
+			return oi, err
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(size))
 	}
-	n, err := c.bitrotWriteToCache(cachePath, cacheDataFile, reader, actualSize)
+	n, md5sum, err := c.bitrotWriteToCache(cachePath, cacheDataFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
 		c.setOffline()
 	}
 	if err != nil {
 		removeAll(cachePath)
-		return err
+		return oi, err
 	}
 
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
-		return IncompleteBody{Bucket: bucket, Object: object}
+		return oi, IncompleteBody{Bucket: bucket, Object: object}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
+	if c.commitWriteback {
+		metadata["content-md5"] = md5sum
+		if md5bytes, err := base64.StdEncoding.DecodeString(md5sum); err == nil {
+			metadata["etag"] = hex.EncodeToString(md5bytes)
+		}
+		metadata[writeBackStatusHeader] = CommitPending.String()
+	}
+	return ObjectInfo{
+			Bucket:      bucket,
+			Name:        object,
+			ETag:        metadata["etag"],
+			Size:        n,
+			UserDefined: metadata,
+		},
+		c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
 }
 
 // Caches the range to disk
@@ -751,7 +795,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 
 	}
 	cacheFile := MustGetUUID()
-	n, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
+	n, _, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
 		c.setOffline()
@@ -866,8 +910,8 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 // Get returns ObjectInfo and reader for object from disk cache
 func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	cLock := c.NewNSLockFn(cacheObjPath)
-	if err := cLock.GetRLock(globalOperationTimeout); err != nil {
+	if err := cLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return nil, numHits, err
 	}
 
@@ -918,7 +962,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	}
 	if globalCacheKMS != nil {
 		// clean up internal SSE cache metadata
-		delete(gr.ObjInfo.UserDefined, crypto.SSEHeader)
+		delete(gr.ObjInfo.UserDefined, xhttp.AmzServerSideEncryption)
 	}
 	if !rngInfo.Empty() {
 		// overlay Size with actual object size and not the range size
@@ -930,8 +974,8 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 
 // Deletes the cached object
 func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	cLock := c.NewNSLockFn(cacheObjPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -951,3 +995,36 @@ func (c *diskCache) Exists(ctx context.Context, bucket, object string) bool {
 	}
 	return true
 }
+
+// queues writeback upload failures on server startup
+func (c *diskCache) scanCacheWritebackFailures(ctx context.Context) {
+	defer close(c.retryWritebackCh)
+	filterFn := func(name string, typ os.FileMode) error {
+		if name == minioMetaBucket {
+			// Proceed to next file.
+			return nil
+		}
+		cacheDir := pathJoin(c.dir, name)
+		meta, _, _, err := c.statCachedMeta(ctx, cacheDir)
+		if err != nil {
+			return nil
+		}
+
+		objInfo := meta.ToObjectInfo("", "")
+		status, ok := objInfo.UserDefined[writeBackStatusHeader]
+		if !ok || status == CommitComplete.String() {
+			return nil
+		}
+		select {
+		case c.retryWritebackCh <- objInfo:
+		default:
+		}
+
+		return nil
+	}
+
+	if err := readDirFilterFn(c.dir, filterFn); err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+}

cmd/disk-cache-stats.go

@@ -23,6 +23,10 @@ import (
 // CacheDiskStats represents cache disk statistics
 // such as current disk usage and available.
 type CacheDiskStats struct {
+	// used cache size
+	UsageSize uint64
+	// total cache disk capacity
+	TotalCapacity uint64
 	// indicates if usage is high or low, if high value is '1', if low its '0'
 	UsageState int32
 	// indicates the current usage percentage of this cache disk

cmd/disk-cache.go

@@ -22,24 +22,47 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/cmd/config/cache"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
-	cacheBlkSize    = int64(1 * 1024 * 1024)
+	cacheBlkSize          = 1 << 20
-	cacheGCInterval = time.Minute * 30
+	cacheGCInterval       = time.Minute * 30
+	writeBackStatusHeader = ReservedMetadataPrefixLower + "write-back-status"
+	writeBackRetryHeader  = ReservedMetadataPrefixLower + "write-back-retry"
 )
 
+type cacheCommitStatus string
+
+const (
+	// CommitPending - cache writeback with backend is pending.
+	CommitPending cacheCommitStatus = "pending"
+
+	// CommitComplete - cache writeback completed ok.
+	CommitComplete cacheCommitStatus = "complete"
+
+	// CommitFailed - cache writeback needs a retry.
+	CommitFailed cacheCommitStatus = "failed"
+)
+
+// String returns string representation of status
+func (s cacheCommitStatus) String() string {
+	return string(s)
+}
+
 // CacheStorageInfo - represents total, free capacity of
 // underlying cache storage.
 type CacheStorageInfo struct {
@@ -69,19 +92,22 @@ type cacheObjects struct {
 	exclude []string
 	// number of accesses after which to cache an object
 	after int
+	// commit objects in async manner
+	commitWriteback bool
 	// if true migration is in progress from v1 to v2
 	migrating bool
 	// mutex to protect migration bool
 	migMutex sync.Mutex
-
+	// retry queue for writeback cache mode to reattempt upload to backend
+	wbRetryCh chan ObjectInfo
 	// Cache stats
 	cacheStats *CacheStats
 
-	GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	InnerGetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
-	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerGetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	DeleteObjectFn   func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerDeleteObjectFn   func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerPutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	CopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerCopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
@@ -120,7 +146,7 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 
 // DeleteObject clears cache entry if backend delete operation succeeds
 func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if objInfo, err = c.DeleteObjectFn(ctx, bucket, object, opts); err != nil {
+	if objInfo, err = c.InnerDeleteObjectFn(ctx, bucket, object, opts); err != nil {
 		return
 	}
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
@@ -188,14 +214,14 @@ func (c *cacheObjects) incCacheStats(size int64) {
 
 func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	var cc *cacheControl
 	var cacheObjSize int64
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 
 	cacheReader, numCacheHits, cacheErr := dcache.Get(ctx, bucket, object, rs, h, opts)
@@ -224,14 +250,14 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		if cc != nil && cc.noStore {
 			cacheReader.Close()
 			c.cacheStats.incMiss()
-			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			bReader, err := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 			bReader.ObjInfo.CacheLookupStatus = CacheHit
 			bReader.ObjInfo.CacheStatus = CacheMiss
 			return bReader, err
 		}
 	}
 
-	objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
+	objInfo, err := c.InnerGetObjectInfoFn(ctx, bucket, object, opts)
 	if backendDownError(err) && cacheErr == nil {
 		c.incCacheStats(cacheObjSize)
 		return cacheReader, nil
@@ -255,7 +281,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			cacheReader.Close()
 		}
 		c.cacheStats.incMiss()
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	// skip cache for objects with locks
 	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
@@ -265,7 +291,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			cacheReader.Close()
 		}
 		c.cacheStats.incMiss()
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	if cacheErr == nil {
 		// if ETag matches for stale cache entry, serve from cache
@@ -283,7 +309,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
 
-	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	bkReader, bkErr := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 
 	if bkErr != nil {
 		return bkReader, bkErr
@@ -305,14 +331,12 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return bkReader, bkErr
 	}
 
-	if rs != nil {
+	if rs != nil && !dcache.enableRange {
 		go func() {
 			// if range caching is disabled, download entire object.
-			if !dcache.enableRange {
+			rs = nil
-				rs = nil
-			}
 			// fill cache in the background for range GET requests
-			bReader, bErr := c.GetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
+			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
 				return
 			}
@@ -335,9 +359,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	userDefined := getMetadata(bkReader.ObjInfo)
 	go func() {
-		putErr := dcache.Put(ctx, bucket, object,
+		_, putErr := dcache.Put(ctx, bucket, object,
 			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
-			bkReader.ObjInfo.Size, nil, ObjectOptions{
+			bkReader.ObjInfo.Size, rs, ObjectOptions{
 				UserDefined: userDefined,
 			}, false)
 		// close the write end of the pipe, so the error gets
@@ -351,7 +375,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 // Returns ObjectInfo from cache if available.
 func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
-	getObjectInfoFn := c.GetObjectInfoFn
+	getObjectInfoFn := c.InnerGetObjectInfoFn
 
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
 		return getObjectInfoFn(ctx, bucket, object, opts)
@@ -409,7 +433,7 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // CopyObject reverts to backend after evicting any stale cache entries
 func (c *cacheObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	copyObjectFn := c.CopyObjectFn
+	copyObjectFn := c.InnerCopyObjectFn
 	if c.isCacheExclude(srcBucket, srcObject) || c.skipCache() {
 		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 	}
@@ -596,7 +620,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 
 // PutObject - caches the uploaded object for single Put operations
 func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	putObjectFn := c.PutObjectFn
+	putObjectFn := c.InnerPutObjectFn
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
@@ -631,13 +655,20 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
-
+	if c.commitWriteback {
+		oi, err := dcache.Put(ctx, bucket, object, r, r.Size(), nil, opts, false)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		go c.uploadObject(GlobalContext, oi)
+		return oi, nil
+	}
 	objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
 
 	if err == nil {
 		go func() {
 			// fill cache in the background
-			bReader, bErr := c.GetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
 			if bErr != nil {
 				return
 			}
@@ -649,9 +680,68 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 			}
 		}()
 	}
-
 	return objInfo, err
+}
 
+// upload cached object to backend in async commit mode.
+func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
+	dcache, err := c.getCacheToLoc(ctx, oi.Bucket, oi.Name)
+	if err != nil {
+		// disk cache could not be located.
+		logger.LogIf(ctx, fmt.Errorf("Could not upload %s/%s to backend: %w", oi.Bucket, oi.Name, err))
+		return
+	}
+	cReader, _, bErr := dcache.Get(ctx, oi.Bucket, oi.Name, nil, http.Header{}, ObjectOptions{})
+	if bErr != nil {
+		return
+	}
+	defer cReader.Close()
+
+	if cReader.ObjInfo.ETag != oi.ETag {
+		return
+	}
+	st := cacheCommitStatus(oi.UserDefined[writeBackStatusHeader])
+	if st == CommitComplete || st.String() == "" {
+		return
+	}
+	hashReader, err := hash.NewReader(cReader, oi.Size, "", "", oi.Size, globalCLIContext.StrictS3Compat)
+	if err != nil {
+		return
+	}
+	var opts ObjectOptions
+	opts.UserDefined = make(map[string]string)
+	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
+	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader, nil, nil), opts)
+	wbCommitStatus := CommitComplete
+	if err != nil {
+		wbCommitStatus = CommitFailed
+	}
+
+	meta := cloneMSS(cReader.ObjInfo.UserDefined)
+	retryCnt := 0
+	if wbCommitStatus == CommitFailed {
+		retryCnt, _ = strconv.Atoi(meta[writeBackRetryHeader])
+		retryCnt++
+		meta[writeBackRetryHeader] = strconv.Itoa(retryCnt)
+	} else {
+		delete(meta, writeBackRetryHeader)
+	}
+	meta[writeBackStatusHeader] = wbCommitStatus.String()
+	meta["etag"] = oi.ETag
+	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, objInfo.Size, nil, "", false)
+	if retryCnt > 0 {
+		// slow down retries
+		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
+		c.queueWritebackRetry(oi)
+	}
+}
+
+func (c *cacheObjects) queueWritebackRetry(oi ObjectInfo) {
+	select {
+	case c.wbRetryCh <- oi:
+		c.uploadObject(GlobalContext, oi)
+	default:
+	}
 }
 
 // Returns cacheObjects for use by Server.
@@ -662,25 +752,26 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		return nil, err
 	}
 	c := &cacheObjects{
-		cache:      cache,
+		cache:           cache,
-		exclude:    config.Exclude,
+		exclude:         config.Exclude,
-		after:      config.After,
+		after:           config.After,
-		migrating:  migrateSw,
+		migrating:       migrateSw,
-		migMutex:   sync.Mutex{},
+		migMutex:        sync.Mutex{},
-		cacheStats: newCacheStats(),
+		commitWriteback: config.CommitWriteback,
-		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+		cacheStats:      newCacheStats(),
+		InnerGetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
-		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+		InnerGetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
-		DeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+		InnerDeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+		InnerPutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
-		CopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+		InnerCopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
 	}
@@ -690,6 +781,10 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 			dcache := c.cache[i]
 			cacheDiskStats[i] = CacheDiskStats{}
 			if dcache != nil {
+				info, err := getDiskInfo(dcache.dir)
+				logger.LogIf(ctx, err)
+				cacheDiskStats[i].UsageSize = info.Used
+				cacheDiskStats[i].TotalCapacity = info.Total
 				cacheDiskStats[i].Dir = dcache.stats.Dir
 				atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
 				atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))
@@ -701,6 +796,15 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		go c.migrateCacheFromV1toV2(ctx)
 	}
 	go c.gc(ctx)
+	if c.commitWriteback {
+		c.wbRetryCh = make(chan ObjectInfo, 10000)
+		go func() {
+			<-GlobalContext.Done()
+			close(c.wbRetryCh)
+		}()
+		go c.queuePendingWriteback(ctx)
+	}
+
 	return c, nil
 }
 
@@ -726,3 +830,25 @@ func (c *cacheObjects) gc(ctx context.Context) {
 		}
 	}
 }
+
+// queues any pending or failed async commits when server restarts
+func (c *cacheObjects) queuePendingWriteback(ctx context.Context) {
+	for _, dcache := range c.cache {
+		if dcache != nil {
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case oi, ok := <-dcache.retryWritebackCh:
+					if !ok {
+						goto next
+					}
+					c.queueWritebackRetry(oi)
+				default:
+					time.Sleep(time.Second * 1)
+				}
+			}
+		next:
+		}
+	}
+}

cmd/encryption-v1.go

@@ -386,13 +386,13 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 		header:            h,
 		bucket:            bucket,
 		object:            object,
-		customerKeyHeader: h.Get(crypto.SSECKey),
+		customerKeyHeader: h.Get(xhttp.AmzServerSideEncryptionCustomerKey),
 		copySource:        copySource,
 		metadata:          cloneMSS(oi.UserDefined),
 	}
 
 	if w.copySource {
-		w.customerKeyHeader = h.Get(crypto.SSECopyKey)
+		w.customerKeyHeader = h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	}
 
 	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
@@ -434,12 +434,12 @@ func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
 	var err error
 	if d.copySource {
 		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
+			d.header.Set(xhttp.AmzServerSideEncryptionCopyCustomerKey, d.customerKeyHeader)
 			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
 		}
 	} else {
 		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(crypto.SSECKey, d.customerKeyHeader)
+			d.header.Set(xhttp.AmzServerSideEncryptionCustomerKey, d.customerKeyHeader)
 			key, err = ParseSSECustomerHeader(d.header)
 		}
 	}

cmd/encryption-v1_test.go

@@ -19,12 +19,17 @@ package cmd
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
+	"fmt"
 	"net/http"
+	"os"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/sio"
 )
 
@@ -34,27 +39,27 @@ var encryptRequestTests = []struct {
 }{
 	{
 		header: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			xhttp.AmzServerSideEncryptionCustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{},
 	},
 	{
 		header: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			xhttp.AmzServerSideEncryptionCustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{
-			crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
 		},
 	},
 }
 
 func TestEncryptRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	defer func(flag bool) { globalIsTLS = flag }(globalIsTLS)
-	globalIsSSL = true
+	globalIsTLS = true
 	for i, test := range encryptRequestTests {
 		content := bytes.NewReader(make([]byte, 64))
 		req := &http.Request{Header: http.Header{}}
@@ -66,13 +71,13 @@ func TestEncryptRequest(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
-		if kdf, ok := test.metadata[crypto.SSESealAlgorithm]; !ok {
+		if kdf, ok := test.metadata[crypto.MetaAlgorithm]; !ok {
 			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
 		}
-		if iv, ok := test.metadata[crypto.SSEIV]; !ok {
+		if iv, ok := test.metadata[crypto.MetaIV]; !ok {
 			t.Errorf("Test %d: crypto.SSEIV must be part of metadata: %v", i, iv)
 		}
-		if mac, ok := test.metadata[crypto.SSECSealedKey]; !ok {
+		if mac, ok := test.metadata[crypto.MetaSealedKeySSEC]; !ok {
 			t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
 		}
 	}
@@ -89,33 +94,33 @@ var decryptObjectInfoTests = []struct {
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
-		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
+		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
-		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSECSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaSealedKeySSEC: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
 		request: &http.Request{Header: http.Header{}},
 		expErr:  errEncryptedObject,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
-		request: &http.Request{Method: http.MethodGet, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Method: http.MethodGet, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  errInvalidEncryptionParameters,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
-		request: &http.Request{Method: http.MethodHead, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Method: http.MethodHead, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  errInvalidEncryptionParameters,
 	},
 	{
-		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
+		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
-		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  errObjectTampered,
 	},
 }
@@ -217,10 +222,10 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
 		Name:   "object",
 		Size:   595160760,
 		UserDefined: map[string]string{
-			crypto.SSEMultipart:                    "",
+			crypto.MetaMultipart:                   "",
-			crypto.SSEIV:                           "HTexa=",
+			crypto.MetaIV:                          "HTexa=",
-			crypto.SSESealAlgorithm:                "DAREv2-HMAC-SHA256",
+			crypto.MetaAlgorithm:                   "DAREv2-HMAC-SHA256",
-			crypto.SSECSealedKey:                   "IAA8PGAA==",
+			crypto.MetaSealedKeySSEC:               "IAA8PGAA==",
 			ReservedMetadataPrefix + "actual-size": "594870264",
 			"content-type":                         "application/octet-stream",
 			"etag":                                 "166b1545b4c1535294ee0686678bea8c-2",
@@ -272,11 +277,11 @@ func TestGetDecryptedRange(t *testing.T) {
 		}
 		udMap = func(isMulti bool) map[string]string {
 			m := map[string]string{
-				crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
+				crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm,
-				crypto.SSEMultipart:     "1",
+				crypto.MetaMultipart: "1",
 			}
 			if !isMulti {
-				delete(m, crypto.SSEMultipart)
+				delete(m, crypto.MetaMultipart)
 			}
 			return m
 		}
@@ -549,56 +554,56 @@ var getDefaultOptsTests = []struct {
 	encryptionType encrypt.Type
 	err            error
 }{
-	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
-		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: encrypt.SSEC,
 		err:            nil}, // 0
-	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
-		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     true,
 		metadata:       nil,
 		encryptionType: "",
 		err:            nil}, // 1
-	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
-		crypto.SSECKey:    []string{"Mz"},
+		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"Mz"},
-		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: "",
 		err:            crypto.ErrInvalidCustomerKey}, // 2
-	{headers: http.Header{crypto.SSEHeader: []string{"AES256"}},
+	{headers: http.Header{xhttp.AmzServerSideEncryption: []string{"AES256"}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: encrypt.S3,
 		err:            nil}, // 3
 	{headers: http.Header{},
 		copySource: false,
-		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
-			crypto.S3KMSKeyID:     "kms-key",
+			crypto.MetaKeyID:             "kms-key",
-			crypto.S3KMSSealedKey: "m-key"},
+			crypto.MetaDataEncryptionKey: "m-key"},
 		encryptionType: encrypt.S3,
 		err:            nil}, // 4
 	{headers: http.Header{},
 		copySource: true,
-		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
-			crypto.S3KMSKeyID:     "kms-key",
+			crypto.MetaKeyID:             "kms-key",
-			crypto.S3KMSSealedKey: "m-key"},
+			crypto.MetaDataEncryptionKey: "m-key"},
 		encryptionType: "",
 		err:            nil}, // 5
-	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
-		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     true,
 		metadata:       nil,
 		encryptionType: encrypt.SSEC,
 		err:            nil}, // 6
-	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
-		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: "",
@@ -622,3 +627,89 @@ func TestGetDefaultOpts(t *testing.T) {
 		}
 	}
 }
+func Test_decryptObjectInfo(t *testing.T) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var dst [32]byte
+	for i := range testSet {
+		t.Run(fmt.Sprint("case-", i), func(t *testing.T) {
+			test := &testSet[i]
+			_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func Benchmark_decryptObjectInfo(b *testing.B) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.SetBytes(int64(len(testSet)))
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var dst [32]byte
+			for i := range testSet {
+				test := &testSet[i]
+				_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		}
+	})
+}

cmd/endpoint-ellipses.go

@@ -329,7 +329,7 @@ var (
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (
-	endpointZones EndpointZones, setupType SetupType, err error) {
+	endpointServerPools EndpointServerPools, setupType SetupType, err error) {
 
 	if len(args) == 0 {
 		return nil, -1, errInvalidArgument
@@ -352,34 +352,29 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, err
 		}
-		endpointZones = append(endpointZones, ZoneEndpoints{
+		endpointServerPools = append(endpointServerPools, ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
 		})
 		setupType = newSetupType
-		return endpointZones, setupType, nil
+		return endpointServerPools, setupType, nil
 	}
 
-	var prevSetupType SetupType
 	var foundPrevLocal bool
 	for _, arg := range args {
 		setArgs, err := GetAllSets(uint64(setDriveCount), arg)
 		if err != nil {
 			return nil, -1, err
 		}
-		var endpointList Endpoints
+		endpointList, gotSetupType, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
-		endpointList, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
 		if err != nil {
 			return nil, -1, err
 		}
 		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
-			return nil, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
+			return nil, -1, fmt.Errorf("All serverPools should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
 		}
-		if prevSetupType != UnknownSetupType && prevSetupType != setupType {
+		if err = endpointServerPools.Add(ZoneEndpoints{
-			return nil, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
-		}
-		if err = endpointZones.Add(ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
@@ -390,8 +385,13 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if setDriveCount == 0 {
 			setDriveCount = len(setArgs[0])
 		}
-		prevSetupType = setupType
+		if setupType == UnknownSetupType {
+			setupType = gotSetupType
+		}
+		if setupType == ErasureSetupType && gotSetupType == DistErasureSetupType {
+			setupType = DistErasureSetupType
+		}
 	}
 
-	return endpointZones, setupType, nil
+	return endpointServerPools, setupType, nil
 }

cmd/endpoint.go

@@ -17,7 +17,8 @@
 package cmd
 
 import (
-	"crypto/tls"
+	"context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -26,17 +27,19 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // EndpointType - enum for endpoint type.
@@ -54,7 +57,7 @@ const (
 // See proxyRequest() for details.
 type ProxyEndpoint struct {
 	Endpoint
-	Transport *http.Transport
+	Transport http.RoundTripper
 }
 
 // Endpoint - any type of endpoint.
@@ -201,12 +204,12 @@ type ZoneEndpoints struct {
 	Endpoints    Endpoints
 }
 
-// EndpointZones - list of list of endpoints
+// EndpointServerPools - list of list of endpoints
-type EndpointZones []ZoneEndpoints
+type EndpointServerPools []ZoneEndpoints
 
 // GetLocalZoneIdx returns the zone which endpoint belongs to locally.
 // if ep is remote this code will return -1 zoneIndex
-func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
+func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
 	for i, zep := range l {
 		for _, cep := range zep.Endpoints {
 			if cep.IsLocal && ep.IsLocal {
@@ -220,14 +223,14 @@ func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
 }
 
 // Add add zone endpoints
-func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
+func (l *EndpointServerPools) Add(zeps ZoneEndpoints) error {
 	existSet := set.NewStringSet()
 	for _, zep := range *l {
 		for _, ep := range zep.Endpoints {
 			existSet.Add(ep.String())
 		}
 	}
-	// Validate if there are duplicate endpoints across zones
+	// Validate if there are duplicate endpoints across serverPools
 	for _, ep := range zeps.Endpoints {
 		if existSet.Contains(ep.String()) {
 			return fmt.Errorf("duplicate endpoints found")
@@ -237,18 +240,34 @@ func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
 	return nil
 }
 
+// Localhost - returns the local hostname from list of endpoints
+func (l EndpointServerPools) Localhost() string {
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				u := &url.URL{
+					Scheme: endpoint.Scheme,
+					Host:   endpoint.Host,
+				}
+				return u.String()
+			}
+		}
+	}
+	return ""
+}
+
 // FirstLocal returns true if the first endpoint is local.
-func (l EndpointZones) FirstLocal() bool {
+func (l EndpointServerPools) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
 }
 
 // HTTPS - returns true if secure for URLEndpointType.
-func (l EndpointZones) HTTPS() bool {
+func (l EndpointServerPools) HTTPS() bool {
 	return l[0].Endpoints.HTTPS()
 }
 
 // NEndpoints - returns all nodes count
-func (l EndpointZones) NEndpoints() (count int) {
+func (l EndpointServerPools) NEndpoints() (count int) {
 	for _, ep := range l {
 		count += len(ep.Endpoints)
 	}
@@ -256,7 +275,7 @@ func (l EndpointZones) NEndpoints() (count int) {
 }
 
 // Hostnames - returns list of unique hostnames
-func (l EndpointZones) Hostnames() []string {
+func (l EndpointServerPools) Hostnames() []string {
 	foundSet := set.NewStringSet()
 	for _, ep := range l {
 		for _, endpoint := range ep.Endpoints {
@@ -269,6 +288,52 @@ func (l EndpointZones) Hostnames() []string {
 	return foundSet.ToSlice()
 }
 
+// hostsSorted will return all hosts found.
+// The LOCAL host will be nil, but the indexes of all hosts should
+// remain consistent across the cluster.
+func (l EndpointServerPools) hostsSorted() []*xnet.Host {
+	peers, localPeer := l.peers()
+	sort.Strings(peers)
+	hosts := make([]*xnet.Host, len(peers))
+	for i, hostStr := range peers {
+		if hostStr == localPeer {
+			continue
+		}
+		host, err := xnet.ParseHost(hostStr)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+		hosts[i] = host
+	}
+
+	return hosts
+}
+
+// peers will return all peers, including local.
+// The local peer is returned as a separate string.
+func (l EndpointServerPools) peers() (peers []string, local string) {
+	allSet := set.NewStringSet()
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.Type() != URLEndpointType {
+				continue
+			}
+
+			peer := endpoint.Host
+			if endpoint.IsLocal {
+				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
+					local = peer
+				}
+			}
+
+			allSet.Add(peer)
+		}
+	}
+
+	return allSet.ToSlice(), local
+}
+
 // Endpoints - list of same type of endpoint.
 type Endpoints []Endpoint
 
@@ -286,6 +351,14 @@ func (endpoints Endpoints) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// GetAllStrings - returns allstring of all endpoints
+func (endpoints Endpoints) GetAllStrings() (all []string) {
+	for _, e := range endpoints {
+		all = append(all, e.String())
+	}
+	return
+}
+
 func hostResolveToLocalhost(endpoint Endpoint) bool {
 	hostIPs, err := getHostIP(endpoint.Hostname())
 	if err != nil {
@@ -638,14 +711,14 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 
 	// All endpoints are pointing to local host
 	if len(endpoints) == localEndpointCount {
-		// If all endpoints have same port number, Just treat it as distErasure setup
+		// If all endpoints have same port number, Just treat it as local erasure setup
 		// using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerHostSet) > 1 {
 				return endpoints, setupType,
 					config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
 			}
-			return endpoints, DistErasureSetupType, nil
+			return endpoints, ErasureSetupType, nil
 		}
 
 		// Even though all endpoints are local, but those endpoints use different ports.
@@ -688,9 +761,9 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 // the first element from the set of peers which indicate that
 // they are local. There is always one entry that is local
 // even with repeated server endpoints.
-func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
+func GetLocalPeer(endpointServerPools EndpointServerPools) (localPeer string) {
 	peerSet := set.NewStringSet()
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -712,28 +785,6 @@ func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
 	return peerSet.ToSlice()[0]
 }
 
-// GetRemotePeers - get hosts information other than this minio service.
-func GetRemotePeers(endpointZones EndpointZones) []string {
-	peerSet := set.NewStringSet()
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			if endpoint.Type() != URLEndpointType {
-				continue
-			}
-
-			peer := endpoint.Host
-			if endpoint.IsLocal {
-				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
-					continue
-				}
-			}
-
-			peerSet.Add(peer)
-		}
-	}
-	return peerSet.ToSlice()
-}
-
 // GetProxyEndpointLocalIndex returns index of the local proxy endpoint
 func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	for i, pep := range proxyEps {
@@ -744,13 +795,79 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	return -1
 }
 
+func httpDo(clnt *http.Client, req *http.Request, f func(*http.Response, error) error) error {
+	ctx, cancel := context.WithTimeout(GlobalContext, 200*time.Millisecond)
+	defer cancel()
+
+	// Run the HTTP request in a goroutine and pass the response to f.
+	c := make(chan error, 1)
+	req = req.WithContext(ctx)
+	go func() { c <- f(clnt.Do(req)) }()
+	select {
+	case <-ctx.Done():
+		<-c // Wait for f to return.
+		return ctx.Err()
+	case err := <-c:
+		return err
+	}
+}
+
+func getOnlineProxyEndpointIdx() int {
+	type reqIndex struct {
+		Request *http.Request
+		Idx     int
+	}
+
+	proxyRequests := make(map[*http.Client]reqIndex, len(globalProxyEndpoints))
+	for i, proxyEp := range globalProxyEndpoints {
+		proxyEp := proxyEp
+		serverURL := &url.URL{
+			Scheme: proxyEp.Scheme,
+			Host:   proxyEp.Host,
+			Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
+		}
+
+		req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
+		if err != nil {
+			continue
+		}
+
+		proxyRequests[&http.Client{
+			Transport: proxyEp.Transport,
+		}] = reqIndex{
+			Request: req,
+			Idx:     i,
+		}
+	}
+
+	for c, r := range proxyRequests {
+		if err := httpDo(c, r.Request, func(resp *http.Response, err error) error {
+			if err != nil {
+				return err
+			}
+			xhttp.DrainBody(resp.Body)
+			if resp.StatusCode != http.StatusOK {
+				return errors.New(resp.Status)
+			}
+			if v := resp.Header.Get(xhttp.MinIOServerStatus); v == unavailable {
+				return errors.New(v)
+			}
+			return nil
+		}); err != nil {
+			continue
+		}
+		return r.Idx
+	}
+	return -1
+}
+
 // GetProxyEndpoints - get all endpoints that can be used to proxy list request.
-func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
+func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint {
 	var proxyEps []ProxyEndpoint
 
 	proxyEpSet := set.NewStringSet()
 
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -762,27 +879,13 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
 			}
 			proxyEpSet.Add(host)
 
-			var tlsConfig *tls.Config
-			if globalIsSSL {
-				tlsConfig = &tls.Config{
-					ServerName: endpoint.Hostname(),
-					RootCAs:    globalRootCAs,
-				}
-			}
-
-			tr := newCustomHTTPTransport(tlsConfig, rest.DefaultTimeout)()
-			// Allow more requests to be in flight with higher response header timeout.
-			tr.ResponseHeaderTimeout = 30 * time.Minute
-			tr.MaxIdleConns = 64
-			tr.MaxIdleConnsPerHost = 64
-
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,
-				Transport: tr,
+				Transport: globalProxyTransport,
 			})
 		}
 	}
-	return proxyEps, nil
+	return proxyEps
 }
 
 func updateDomainIPs(endPoints set.StringSet) {

cmd/endpoint_test.go

@@ -246,7 +246,7 @@ func TestCreateEndpoints(t *testing.T) {
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d4"}, IsLocal: true},
-		}, DistErasureSetupType, nil},
+		}, ErasureSetupType, nil},
 		// DistErasure Setup with URLEndpointType having mixed naming to local host.
 		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 
@@ -380,24 +380,28 @@ func TestGetRemotePeers(t *testing.T) {
 	testCases := []struct {
 		endpointArgs   []string
 		expectedResult []string
+		expectedLocal  string
 	}{
-		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
+		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}, ""},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000", "localhost:9000"}, "localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9000", "localhost:9001", "localhost:9002", "localhost:9003"}, "localhost:9000"},
 	}
 
 	for _, testCase := range testCases {
 		zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
+				t.Errorf("error: expected = <nil>, got = %v", err)
 			}
 		}
-		remotePeers := GetRemotePeers(zendpoints)
+		remotePeers, local := zendpoints.peers()
 		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
-			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+			t.Errorf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+		}
+		if local != testCase.expectedLocal {
+			t.Errorf("expected: %v, got: %v", testCase.expectedLocal, local)
 		}
 	}
 }