Fix missing GetObjectNInfo Closure (#11243)

Review for missing Close of returned value from `GetObjectNInfo`.

This was often obscured by the stuff that auto-unlocks when reaching EOF.

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,9 +10,10 @@
 ## Types of changes
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
+- [ ] Optimization (provides speedup with no functional changes)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 
 ## Checklist:
 - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
-- [ ] Documentation needed
-- [ ] Unit tests needed
+- [ ] Documentation updated
+- [ ] Unit tests added/updated

.github/workflows/go.yml

@@ -26,7 +26,6 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_CI_CD: 1
         run: |
           go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
           go test -v --timeout 50m ./...
@@ -35,7 +34,6 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_CI_CD: 1
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0

.golangci.yml

@@ -27,6 +27,7 @@ issues:
 run:
   skip-dirs:
     - pkg/rpc
+    - pkg/argon2
 
 service:
   golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

Dockerfile

@@ -11,10 +11,12 @@ RUN  \
      git clone https://github.com/minio/minio && cd minio && \
      git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
@@ -22,11 +24,14 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=builder /go/bin/minio /usr/bin/minio
-COPY --from=builder /go/minio/CREDITS /third_party/
+COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
+COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
 COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     microdnf clean all && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

Dockerfile.cicd

@@ -7,30 +7,36 @@ ENV CGO_ENABLED 0
 ENV GO111MODULE on
 
 RUN  \
-  apk add --no-cache git && \
-  git clone https://github.com/minio/minio && cd minio && \
-  git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+    apk add --no-cache git && \
+    git clone https://github.com/minio/minio && cd minio && \
+    git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
+
+ARG TARGETARCH
 
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-  MINIO_SECRET_KEY_FILE=secret_key \
-  MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-  MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
-  MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 EXPOSE 9000
 
 COPY --from=builder /go/bin/minio /usr/bin/minio
-COPY --from=builder /go/minio/CREDITS /third_party/
+COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
+COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
 COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-  apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
-  echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
+    microdnf update --nodocs && \
+    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+    microdnf clean all
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-CMD ["minio", "server", "/data"]
+CMD ["minio", "server", "/data"]

Dockerfile.dev

@@ -1,22 +1,28 @@
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
+
+ARG TARGETARCH
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
-COPY CREDITS /third_party/
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
 
-ENV MINIO_UPDATE off
-ENV MINIO_ACCESS_KEY_FILE=access_key \
+ENV MINIO_UPDATE=off \
+    MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
-RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh
+RUN  \
+    microdnf update --nodocs && \
+    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+    microdnf clean all && \
+    chmod +x /usr/bin/minio  && \
+    chmod +x /usr/bin/docker-entrypoint.sh
 
 EXPOSE 9000
 

Dockerfile.dev.browser

@@ -1,4 +1,4 @@
-FROM ubuntu
+FROM ubuntu:20.04
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Dockerfile.release

@@ -1,11 +1,19 @@
-FROM alpine:3.12
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
 
 ARG TARGETARCH
 
-LABEL maintainer="MinIO Inc <dev@min.io>"
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="RELEASE.2021-01-05T05-22-38Z" \
+      release="RELEASE.2021-01-05T05-22-38Z" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
 
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
     MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
@@ -16,16 +24,17 @@ COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
 
 RUN \
-     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
-     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+     microdnf install minisign --nodocs && \
      curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio -o /usr/bin/minio && \
      curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.sha256sum -o /usr/bin/minio.sha256sum && \
      curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.minisig -o /usr/bin/minio.minisig && \
+     microdnf clean all && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

Dockerfile.release.ubi

@@ -1,49 +0,0 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
-
-ARG TARGETARCH
-
-LABEL name="MinIO" \
-      vendor="MinIO Inc <dev@min.io>" \
-      maintainer="MinIO Inc <dev@min.io>" \
-      version="RELEASE.2020-11-25T22-36-25Z" \
-      release="RELEASE.2020-11-25T22-36-25Z" \
-      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
-      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" 
-
-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.ubi.sh /usr/bin/docker-entrypoint.ubi.sh
-COPY CREDITS /licenses/CREDITS
-COPY LICENSE /licenses/LICENSE
-
-RUN \
-     microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils --nodocs && \
-     curl -s -q https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm -o epel-release.rpm && \
-     rpm -ivh epel-release.rpm && microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/minio.minisig -o /usr/bin/minio.minisig && \
-     microdnf clean all && \
-     chmod +x /usr/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.ubi.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     groupadd --gid 1000 minio && \
-     useradd -M --uid 1000 --gid 1000 --home /usr/share/minio minio && \
-     mkdir -p /data && chown -R minio:minio /usr/bin /data
-
-EXPOSE 9000
-
-USER minio
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.ubi.sh"]
-
-VOLUME ["/data"]
-
-CMD ["minio"]

Makefile

@@ -17,9 +17,9 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
-	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
-	@which msgp 1>/dev/null || (echo "Installing msgp" && GO111MODULE=off go get github.com/tinylib/msgp)
-	@which stringer 1>/dev/null || (echo "Installing stringer" && GO111MODULE=off go get golang.org/x/tools/cmd/stringer)
+	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1)
+	@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -57,7 +57,7 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
@@ -71,6 +71,10 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
+hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
+hotfix: install
+
 docker: checks
 	@echo "Building minio docker image '$(TAG)'"
 	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

README.md

@@ -9,16 +9,16 @@ MinIO is a High Performance Object Storage released under Apache License v2.0. I
 ### Stable
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   minio/minio server /data
 ```
 
 ### Edge
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   minio/minio:edge server /data
 ```
 
@@ -150,7 +150,7 @@ service iptables restart
 ```
 
 ## Test using MinIO Browser
-MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 ensure your server has started successfully.
+MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure your server has started successfully.
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 

README_zh_CN.md

@@ -8,16 +8,16 @@ MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合
 ### 稳定版
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   minio/minio server /data
 ```
 
 ### 尝鲜版
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   minio/minio:edge server /data
 ```
 

VULNERABILITY_REPORT.md

@@ -0,0 +1,39 @@
+## Vulnerability Management Policy
+
+This document formally describes the process of addressing and managing a
+reported vulnerability that has been found in the MinIO server code base,
+any directly connected ecosystem component or a direct / indirect dependency
+of the code base.
+
+### Scope
+
+The vulnerability management policy described in this document covers the
+process of investigating, assessing and resolving a vulnerability report
+opened by a MinIO employee or an external third party.
+
+Therefore, it lists pre-conditions and actions that should be performed to
+resolve and fix a reported vulnerability.
+
+### Vulnerability Management Process
+
+The vulnerability management process requires that the vulnerability report
+contains the following information:
+
+ - The project / component that contains the reported vulnerability.
+ - A description of the vulnerability. In particular, the type of the
+   reported vulnerability and how it might be exploited. Alternatively,
+   a well-established vulnerability identifier, e.g. CVE number, can be
+   used instead.
+
+Based on the description mentioned above, a MinIO engineer or security team
+member investigates:
+
+ - Whether the reported vulnerability exists.
+ - The conditions that are required such that the vulnerability can be exploited.
+ - The steps required to fix the vulnerability.
+
+In general, if the vulnerability exists in one of the MinIO code bases
+itself - not in a code dependency - then MinIO will, if possible, fix
+the vulnerability or implement reasonable countermeasures such that the
+vulnerability cannot be exploited anymore.
+

browser/package-lock.json

@@ -2271,7 +2271,9 @@
     "asap": {
       "version": "2.0.6",
       "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
+      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=",
+      "dev": true,
+      "optional": true
     },
     "asn1": {
       "version": "0.2.4",
@@ -2283,19 +2285,20 @@
       }
     },
     "asn1.js": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",
-      "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
+      "version": "5.4.1",
+      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
+      "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
       "requires": {
         "bn.js": "^4.0.0",
         "inherits": "^2.0.1",
-        "minimalistic-assert": "^1.0.0"
+        "minimalistic-assert": "^1.0.0",
+        "safer-buffer": "^2.1.0"
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -3541,9 +3544,9 @@
       }
     },
     "base64-js": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
-      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
+      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
     },
     "batch": {
       "version": "0.6.1",
@@ -3574,13 +3577,12 @@
     "bluebird": {
       "version": "3.7.2",
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==",
-      "dev": true
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
     "bn.js": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.1.tgz",
-      "integrity": "sha512-IUTD/REb78Z2eodka1QZyyEk66pciRcP6Sroka0aI3tG/iwIdYLrBD62RsubR7vqdt3WyX8p4jxeatzmRSphtA=="
+      "version": "5.1.3",
+      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz",
+      "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ=="
     },
     "body-parser": {
       "version": "1.19.0",
@@ -3754,31 +3756,24 @@
       }
     },
     "browserify-rsa": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
-      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz",
+      "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==",
       "requires": {
-        "bn.js": "^4.1.0",
+        "bn.js": "^5.0.0",
         "randombytes": "^2.0.1"
-      },
-      "dependencies": {
-        "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
-        }
       }
     },
     "browserify-sign": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.0.tgz",
-      "integrity": "sha512-hEZC1KEeYuoHRqhGhTy6gWrpJA3ZDjFWv0DE61643ZnOXAKJb3u7yWcrU0mMc9SwAqK1n7myPGndkp0dFG7NFA==",
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz",
+      "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==",
       "requires": {
         "bn.js": "^5.1.1",
         "browserify-rsa": "^4.0.1",
         "create-hash": "^1.2.0",
         "create-hmac": "^1.1.7",
-        "elliptic": "^6.5.2",
+        "elliptic": "^6.5.3",
         "inherits": "^2.0.4",
         "parse-asn1": "^5.1.5",
         "readable-stream": "^3.6.0",
@@ -4022,9 +4017,9 @@
       }
     },
     "chokidar": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.0.tgz",
-      "integrity": "sha512-aXAaho2VJtisB/1fg1+3nlLJqGOuewTzQpd/Tz0yTg2R0e4IGtshYvtjowyEumcBv2z+y4+kc75Mz7j5xJskcQ==",
+      "version": "3.4.3",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.3.tgz",
+      "integrity": "sha512-DtM3g7juCXQxFVSNPNByEC2+NImtBuxQQvWlHunpJIS5Ocr0lG306cC7FCi7cEA0fzmybPUIl4txBIobk1gGOQ==",
       "optional": true,
       "requires": {
         "anymatch": "~3.1.1",
@@ -4034,7 +4029,7 @@
         "is-binary-path": "~2.1.0",
         "is-glob": "~4.0.1",
         "normalize-path": "~3.0.0",
-        "readdirp": "~3.4.0"
+        "readdirp": "~3.5.0"
       },
       "dependencies": {
         "anymatch": {
@@ -4048,9 +4043,9 @@
           }
         },
         "binary-extensions": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz",
-          "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
+          "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
           "optional": true
         },
         "braces": {
@@ -4123,9 +4118,9 @@
           "optional": true
         },
         "readdirp": {
-          "version": "3.4.0",
-          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
-          "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+          "version": "3.5.0",
+          "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz",
+          "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==",
           "optional": true,
           "requires": {
             "picomatch": "^2.2.1"
@@ -4143,9 +4138,9 @@
       }
     },
     "chownr": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
-      "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g=="
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
+      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
     },
     "chrome-trace-event": {
       "version": "1.0.2",
@@ -4713,18 +4708,18 @@
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
     "create-ecdh": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz",
-      "integrity": "sha512-GbEHQPMOswGpKXM9kCWVrremUcBmjteUaQ01T9rkKCPDXfUHX0IoP9LpHYo2NPFampa4e+/pFDc3jQdxrxQLaw==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz",
+      "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==",
       "requires": {
         "bn.js": "^4.1.0",
-        "elliptic": "^6.0.0"
+        "elliptic": "^6.5.3"
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -4925,9 +4920,9 @@
       }
     },
     "cyclist": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-0.2.2.tgz",
-      "integrity": "sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz",
+      "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk="
     },
     "dashdash": {
       "version": "1.14.1",
@@ -5242,9 +5237,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -5420,9 +5415,9 @@
       }
     },
     "duplexify": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.1.tgz",
-      "integrity": "sha512-vM58DwdnKmty+FSPzT14K9JXb90H+j5emaR4KYbr2KTIz00WHGbWOe5ghQTx233ZCLZtrGDALzKwcjEtSt35mA==",
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
+      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
       "requires": {
         "end-of-stream": "^1.0.0",
         "inherits": "^2.0.1",
@@ -5471,9 +5466,9 @@
       "dev": true
     },
     "elliptic": {
-      "version": "6.5.2",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz",
-      "integrity": "sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw==",
+      "version": "6.5.3",
+      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz",
+      "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==",
       "requires": {
         "bn.js": "^4.4.0",
         "brorand": "^1.0.1",
@@ -5485,9 +5480,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -5509,14 +5504,6 @@
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=",
       "dev": true
     },
-    "encoding": {
-      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
-      "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
-      "requires": {
-        "iconv-lite": "~0.4.13"
-      }
-    },
     "end-of-stream": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
@@ -5526,9 +5513,9 @@
       }
     },
     "enhanced-resolve": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.1.tgz",
-      "integrity": "sha512-98p2zE+rL7/g/DzMHMTF4zZlCgeVdJ7yr6xzEpJRYwFYrGi9ANdn5DnJURg6RpBkyk60XYDnWIv51VfIhfNGuA==",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz",
+      "integrity": "sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==",
       "requires": {
         "graceful-fs": "^4.1.2",
         "memory-fs": "^0.5.0",
@@ -5886,9 +5873,9 @@
       },
       "dependencies": {
         "lodash": {
-          "version": "4.17.15",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         },
         "react-is": {
@@ -6162,11 +6149,18 @@
       "dev": true
     },
     "esrecurse": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
-      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+      "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
       "requires": {
-        "estraverse": "^4.1.0"
+        "estraverse": "^5.2.0"
+      },
+      "dependencies": {
+        "estraverse": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+          "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ=="
+        }
       }
     },
     "estraverse": {
@@ -6193,9 +6187,9 @@
       "dev": true
     },
     "events": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz",
-      "integrity": "sha512-Rv+u8MLHNOdMjTAFeT3nCjHn2aGlx435FP/sDHNaRhDEMwyI/aB22Kj2qIN8R0cw3z28psEQLYwxVKLsKrMgWg=="
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz",
+      "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg=="
     },
     "eventsource": {
       "version": "1.0.7",
@@ -6722,27 +6716,6 @@
         "bser": "2.1.1"
       }
     },
-    "fbjs": {
-      "version": "0.8.16",
-      "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.8.16.tgz",
-      "integrity": "sha1-XmdDL1UNxBtXK/VYR7ispk5TN9s=",
-      "requires": {
-        "core-js": "^1.0.0",
-        "isomorphic-fetch": "^2.1.1",
-        "loose-envify": "^1.0.0",
-        "object-assign": "^4.1.0",
-        "promise": "^7.1.1",
-        "setimmediate": "^1.0.5",
-        "ua-parser-js": "^0.7.9"
-      },
-      "dependencies": {
-        "core-js": {
-          "version": "1.2.7",
-          "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz",
-          "integrity": "sha1-ZSKUwUZR2yj6k70tX/KYOk8IxjY="
-        }
-      }
-    },
     "figgy-pudding": {
       "version": "3.5.2",
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
@@ -7013,12 +6986,12 @@
       }
     },
     "flush-write-stream": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz",
-      "integrity": "sha512-calZMC10u0FMUqoiunI2AiGIIUtUIvifNwkHhNupZH4cbNnW1Itkoh/Nf5HFYmDrwWPjrUxpkZT0KhuCq0jmGw==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
+      "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==",
       "requires": {
-        "inherits": "^2.0.1",
-        "readable-stream": "^2.0.4"
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.3.6"
       }
     },
     "follow-redirects": {
@@ -8376,9 +8349,9 @@
           }
         },
         "lodash": {
-          "version": "4.17.15",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         },
         "tapable": {
@@ -8651,14 +8624,6 @@
       "resolved": "https://registry.npmjs.org/humanize/-/humanize-0.0.9.tgz",
       "integrity": "sha1-GZT/rs3+nEQe0r2sdFK3u0yeQaQ="
     },
-    "iconv-lite": {
-      "version": "0.4.23",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz",
-      "integrity": "sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==",
-      "requires": {
-        "safer-buffer": ">= 2.1.2 < 3"
-      }
-    },
     "icss-utils": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-4.1.1.tgz",
@@ -8677,9 +8642,9 @@
       }
     },
     "ieee754": {
-      "version": "1.1.13",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
-      "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
+      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
     },
     "iferr": {
       "version": "0.1.5",
@@ -8746,9 +8711,9 @@
       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
     },
     "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
       "dev": true
     },
     "internal-ip": {
@@ -9013,7 +8978,8 @@
     "is-stream": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
+      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
+      "dev": true
     },
     "is-subset": {
       "version": "0.1.1",
@@ -9068,15 +9034,6 @@
       "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
       "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
     },
-    "isomorphic-fetch": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz",
-      "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=",
-      "requires": {
-        "node-fetch": "^1.0.1",
-        "whatwg-fetch": ">=0.10.0"
-      }
-    },
     "isstream": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
@@ -10573,9 +10530,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.14",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz",
-      "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==",
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
       "dev": true
     },
     "lodash._baseisequal": {
@@ -10682,6 +10639,14 @@
         "tslib": "^1.10.0"
       }
     },
+    "lru-cache": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
+      "requires": {
+        "yallist": "^3.0.2"
+      }
+    },
     "make-dir": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
@@ -10868,9 +10833,9 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
@@ -10993,6 +10958,23 @@
         }
       }
     },
+    "mississippi": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
+      "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
+      "requires": {
+        "concat-stream": "^1.5.0",
+        "duplexify": "^3.4.2",
+        "end-of-stream": "^1.1.0",
+        "flush-write-stream": "^1.0.0",
+        "from2": "^2.1.0",
+        "parallel-transform": "^1.1.0",
+        "pump": "^3.0.0",
+        "pumpify": "^1.3.3",
+        "stream-each": "^1.1.0",
+        "through2": "^2.0.0"
+      }
+    },
     "mixin-deep": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
@@ -11165,19 +11147,10 @@
         "tslib": "^1.10.0"
       }
     },
-    "node-fetch": {
-      "version": "1.6.3",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.6.3.tgz",
-      "integrity": "sha1-3CNO3WSJmC1Y6PDbT2lQKavNjAQ=",
-      "requires": {
-        "encoding": "^0.1.11",
-        "is-stream": "^1.0.1"
-      }
-    },
     "node-forge": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.0.tgz",
-      "integrity": "sha512-7ASaDa3pD+lJ3WvXFsxekJQelBKRpne+GOVbLbtHYdd7pFspyeuJHnWfLplGf3SwKGbfs/aYl5V/JCIaHVUKKQ==",
+      "version": "0.10.0",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",
+      "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==",
       "dev": true
     },
     "node-int64": {
@@ -11768,11 +11741,11 @@
       "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
     },
     "parallel-transform": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz",
-      "integrity": "sha1-1BDwZbBdojCB/NEPKIVMKb2jOwY=",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz",
+      "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==",
       "requires": {
-        "cyclist": "~0.2.2",
+        "cyclist": "^1.0.1",
         "inherits": "^2.0.3",
         "readable-stream": "^2.1.5"
       }
@@ -11788,13 +11761,12 @@
       }
     },
     "parse-asn1": {
-      "version": "5.1.5",
-      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.5.tgz",
-      "integrity": "sha512-jkMYn1dcJqF6d5CpU689bq7w/b5ALS9ROVSpQDPrZsqqesUJii9qutvoT5ltGedNXMO2e16YUWIghG9KxaViTQ==",
+      "version": "5.1.6",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz",
+      "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==",
       "requires": {
-        "asn1.js": "^4.0.0",
+        "asn1.js": "^5.2.0",
         "browserify-aes": "^1.0.0",
-        "create-hash": "^1.1.0",
         "evp_bytestokey": "^1.0.0",
         "pbkdf2": "^3.0.3",
         "safe-buffer": "^5.1.1"
@@ -11913,9 +11885,9 @@
       }
     },
     "pbkdf2": {
-      "version": "3.0.17",
-      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.17.tgz",
-      "integrity": "sha512-U/il5MsrZp7mGg3mSQfn742na2T+1/vHDCG5/iTI3X9MKUuYUZVLQhyRsg06mCgDBTd57TxzgZt7P+fYfjRLtA==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
+      "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==",
       "requires": {
         "create-hash": "^1.1.2",
         "create-hmac": "^1.1.4",
@@ -12218,6 +12190,8 @@
       "version": "7.3.1",
       "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
       "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
+      "dev": true,
+      "optional": true,
       "requires": {
         "asap": "~2.0.3"
       }
@@ -12238,13 +12212,23 @@
       }
     },
     "prop-types": {
-      "version": "15.6.0",
-      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.0.tgz",
-      "integrity": "sha1-zq8IMCL8RrSjX2nhPvda7Q1jmFY=",
+      "version": "15.7.2",
+      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
+      "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
       "requires": {
-        "fbjs": "^0.8.16",
-        "loose-envify": "^1.3.1",
-        "object-assign": "^4.1.1"
+        "loose-envify": "^1.4.0",
+        "object-assign": "^4.1.1",
+        "react-is": "^16.8.1"
+      },
+      "dependencies": {
+        "loose-envify": {
+          "version": "1.4.0",
+          "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
+          "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
+          "requires": {
+            "js-tokens": "^3.0.0 || ^4.0.0"
+          }
+        }
       }
     },
     "prop-types-exact": {
@@ -12330,16 +12314,16 @@
       },
       "dependencies": {
         "bn.js": {
-          "version": "4.11.8",
-          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-          "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
+          "version": "4.11.9",
+          "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz",
+          "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw=="
         }
       }
     },
     "pump": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
-      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
+      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
       "requires": {
         "end-of-stream": "^1.1.0",
         "once": "^1.3.1"
@@ -12353,6 +12337,17 @@
         "duplexify": "^3.6.0",
         "inherits": "^2.0.3",
         "pump": "^2.0.0"
+      },
+      "dependencies": {
+        "pump": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
+          "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
+          "requires": {
+            "end-of-stream": "^1.1.0",
+            "once": "^1.3.1"
+          }
+        }
       }
     },
     "punycode": {
@@ -13492,9 +13487,9 @@
       },
       "dependencies": {
         "lodash": {
-          "version": "4.17.15",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-          "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==",
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
           "dev": true
         }
       }
@@ -13958,12 +13953,12 @@
       "dev": true
     },
     "selfsigned": {
-      "version": "1.10.7",
-      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.7.tgz",
-      "integrity": "sha512-8M3wBCzeWIJnQfl43IKwOmC4H/RAp50S8DF60znzjW5GVqTcSe2vWclt7hmYVPkKPlHWOu5EaWOMZ2Y6W8ZXTA==",
+      "version": "1.10.8",
+      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.8.tgz",
+      "integrity": "sha512-2P4PtieJeEwVgTU9QEcwIRDQ/mXJLX8/+I3ur+Pg16nS8oNbrGxEso9NyYWy8NAmXiNl4dlAp5MwoNeCWzON4w==",
       "dev": true,
       "requires": {
-        "node-forge": "0.9.0"
+        "node-forge": "^0.10.0"
       }
     },
     "semver": {
@@ -14001,10 +13996,13 @@
       }
     },
     "serialize-javascript": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.0.0.tgz",
-      "integrity": "sha512-skZcHYw2vEX4bw90nAr2iTTsz6x2SrHEnfxgKYmZlvJYBEZrvbKtobJWlQ20zczKb3bsHHXXTYt48zBA7ni9cw==",
-      "dev": true
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-3.1.0.tgz",
+      "integrity": "sha512-JIJT1DGiWmIKhzRsG91aS6Ze4sFUrYbltlkg2onR5OrnNM02Kl/hnY/T4FN2omvyeBbQmMJv+K4cPOpGzOTFBg==",
+      "dev": true,
+      "requires": {
+        "randombytes": "^2.1.0"
+      }
     },
     "serializerr": {
       "version": "1.0.3",
@@ -14644,9 +14642,9 @@
       }
     },
     "stream-shift": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
-      "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
+      "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ=="
     },
     "strict-uri-encode": {
       "version": "2.0.0",
@@ -15356,26 +15354,21 @@
       }
     },
     "terser-webpack-plugin": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.3.tgz",
-      "integrity": "sha512-QMxecFz/gHQwteWwSo5nTc6UaICqN1bMedC5sMtUc7y3Ha3Q8y6ZO0iCR8pq4RJC8Hjf0FEPEHZqcMB/+DFCrA==",
+      "version": "1.4.5",
+      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz",
+      "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==",
       "requires": {
         "cacache": "^12.0.2",
         "find-cache-dir": "^2.1.0",
         "is-wsl": "^1.1.0",
         "schema-utils": "^1.0.0",
-        "serialize-javascript": "^2.1.2",
+        "serialize-javascript": "^4.0.0",
         "source-map": "^0.6.1",
         "terser": "^4.1.2",
         "webpack-sources": "^1.4.0",
         "worker-farm": "^1.7.0"
       },
       "dependencies": {
-        "bluebird": {
-          "version": "3.7.2",
-          "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
-          "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
-        },
         "cacache": {
           "version": "12.0.4",
           "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz",
@@ -15443,14 +15436,6 @@
             "path-exists": "^3.0.0"
           }
         },
-        "lru-cache": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
-          "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-          "requires": {
-            "yallist": "^3.0.2"
-          }
-        },
         "make-dir": {
           "version": "2.1.0",
           "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
@@ -15460,23 +15445,6 @@
             "semver": "^5.6.0"
           }
         },
-        "mississippi": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz",
-          "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==",
-          "requires": {
-            "concat-stream": "^1.5.0",
-            "duplexify": "^3.4.2",
-            "end-of-stream": "^1.1.0",
-            "flush-write-stream": "^1.0.0",
-            "from2": "^2.1.0",
-            "parallel-transform": "^1.1.0",
-            "pump": "^3.0.0",
-            "pumpify": "^1.3.3",
-            "stream-each": "^1.1.0",
-            "through2": "^2.0.0"
-          }
-        },
         "p-limit": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
@@ -15511,24 +15479,13 @@
             "find-up": "^3.0.0"
           }
         },
-        "pump": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-          "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-          "requires": {
-            "end-of-stream": "^1.1.0",
-            "once": "^1.3.1"
-          }
-        },
         "serialize-javascript": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-2.1.2.tgz",
-          "integrity": "sha512-rs9OggEUF0V4jUSecXazOYsLfu7OGK2qIn3c7IPBiffz32XniEp/TX9Xmc9LQfK2nQ2QKHvZ2oygKUGU0lG4jQ=="
-        },
-        "source-list-map": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz",
+          "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==",
+          "requires": {
+            "randombytes": "^2.1.0"
+          }
         },
         "source-map": {
           "version": "0.6.1",
@@ -15542,15 +15499,6 @@
           "requires": {
             "figgy-pudding": "^3.5.1"
           }
-        },
-        "webpack-sources": {
-          "version": "1.4.3",
-          "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
-          "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-          "requires": {
-            "source-list-map": "^2.0.0",
-            "source-map": "~0.6.1"
-          }
         }
       }
     },
@@ -15589,9 +15537,9 @@
       "dev": true
     },
     "timers-browserify": {
-      "version": "2.0.11",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz",
-      "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==",
+      "version": "2.0.12",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz",
+      "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==",
       "requires": {
         "setimmediate": "^1.0.4"
       }
@@ -15790,11 +15738,6 @@
       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
       "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
     },
-    "ua-parser-js": {
-      "version": "0.7.20",
-      "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.20.tgz",
-      "integrity": "sha512-8OaIKfzL5cpx8eCMAhhvTlft8GYF8b2eQr6JkCyVdrgjcytyOmPCXrqXFcUnhonRpLlh5yxEZVohm6mzaowUOw=="
-    },
     "uglify-js": {
       "version": "3.9.3",
       "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.9.3.tgz",
@@ -16172,20 +16115,20 @@
       }
     },
     "watchpack": {
-      "version": "1.7.2",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.2.tgz",
-      "integrity": "sha512-ymVbbQP40MFTp+cNMvpyBpBtygHnPzPkHqoIwRRj/0B8KhqQwV8LaKjtbaxF2lK4vl8zN9wCxS46IFCU5K4W0g==",
+      "version": "1.7.5",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz",
+      "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==",
       "requires": {
-        "chokidar": "^3.4.0",
+        "chokidar": "^3.4.1",
         "graceful-fs": "^4.1.2",
         "neo-async": "^2.5.0",
-        "watchpack-chokidar2": "^2.0.0"
+        "watchpack-chokidar2": "^2.0.1"
       }
     },
     "watchpack-chokidar2": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz",
-      "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz",
+      "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==",
       "optional": true,
       "requires": {
         "chokidar": "^2.1.8"
@@ -16447,9 +16390,9 @@
       "dev": true
     },
     "webpack": {
-      "version": "4.43.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.43.0.tgz",
-      "integrity": "sha512-GW1LjnPipFW2Y78OOab8NJlCflB7EFskMih2AHdvjbpKMeDJqEgSx24cXXXiPS65+WSwVyxtDsJH6jGX2czy+g==",
+      "version": "4.44.2",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.44.2.tgz",
+      "integrity": "sha512-6KJVGlCxYdISyurpQ0IPTklv+DULv05rs2hseIXer6D7KrUicRDLFb4IUM1S6LUAKypPM/nSiVSuv8jHu1m3/Q==",
       "requires": {
         "@webassemblyjs/ast": "1.9.0",
         "@webassemblyjs/helper-module-context": "1.9.0",
@@ -16459,7 +16402,7 @@
         "ajv": "^6.10.2",
         "ajv-keywords": "^3.4.1",
         "chrome-trace-event": "^1.0.2",
-        "enhanced-resolve": "^4.1.0",
+        "enhanced-resolve": "^4.3.0",
         "eslint-scope": "^4.0.3",
         "json-parse-better-errors": "^1.0.2",
         "loader-runner": "^2.4.0",
@@ -16472,14 +16415,14 @@
         "schema-utils": "^1.0.0",
         "tapable": "^1.1.3",
         "terser-webpack-plugin": "^1.4.3",
-        "watchpack": "^1.6.1",
+        "watchpack": "^1.7.4",
         "webpack-sources": "^1.4.1"
       },
       "dependencies": {
         "ajv": {
-          "version": "6.12.2",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz",
-          "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==",
+          "version": "6.12.6",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+          "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -16578,9 +16521,9 @@
           }
         },
         "fast-deep-equal": {
-          "version": "3.1.1",
-          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
-          "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA=="
+          "version": "3.1.3",
+          "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+          "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
         },
         "is-accessor-descriptor": {
           "version": "0.1.6",
@@ -16661,29 +16604,10 @@
             "to-regex": "^3.0.2"
           }
         },
-        "source-list-map": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
         "tapable": {
           "version": "1.1.3",
           "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz",
           "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA=="
-        },
-        "webpack-sources": {
-          "version": "1.4.3",
-          "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
-          "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-          "requires": {
-            "source-list-map": "^2.0.0",
-            "source-map": "~0.6.1"
-          }
         }
       }
     },
@@ -17605,7 +17529,6 @@
       "version": "1.4.3",
       "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz",
       "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==",
-      "dev": true,
       "requires": {
         "source-list-map": "^2.0.0",
         "source-map": "~0.6.1"
@@ -17614,14 +17537,12 @@
         "source-list-map": {
           "version": "2.0.1",
           "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
-          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==",
-          "dev": true
+          "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw=="
         },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         }
       }
     },
@@ -17660,11 +17581,6 @@
         }
       }
     },
-    "whatwg-fetch": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz",
-      "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q=="
-    },
     "whatwg-mimetype": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz",
@@ -17780,9 +17696,9 @@
       "dev": true
     },
     "xtend": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
-      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
     },
     "y18n": {
       "version": "4.0.0",

buildscripts/gateway-tests.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-#
-# MinIO Cloud Storage, (C) 2019 MinIO, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-set -E
-set -o pipefail
-
-function start_minio_server()
-{
-    MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
-                    minio --quiet --json server /data --address 127.0.0.1:24242 > server.log 2>&1 &
-    server_pid=$!
-    sleep 10
-
-    echo "$server_pid"
-}
-
-function start_minio_gateway_s3()
-{
-    MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
-                    minio --quiet --json gateway s3 http://127.0.0.1:24242 \
-                    --address 127.0.0.1:24240 > gateway.log 2>&1 &
-    gw_pid=$!
-    sleep 10
-
-    echo "$gw_pid"
-}
-
-function main()
-{
-    sr_pid="$(start_minio_server)"
-    gw_pid="$(start_minio_gateway_s3)"
-
-    SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
-                   aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
-                   healthcheck mc minio-dotnet minio-js \
-                   minio-py s3cmd s3select security
-    rv=$?
-
-    kill "$sr_pid"
-    kill "$gw_pid"
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        echo "=========== Gateway ==========="
-        cat "gateway.log"
-        echo "=========== Server ==========="
-        cat "server.log"
-    fi
-
-    rm -f gateway.log server.log
-}
-
-main "$@"

buildscripts/gen-ldflags.go

@@ -44,10 +44,21 @@ func releaseTag(version string) string {
 		relPrefix = prefix
 	}
 
+	relSuffix := ""
+	if hotfix := os.Getenv("MINIO_HOTFIX"); hotfix != "" {
+		relSuffix = hotfix
+	}
+
 	relTag := strings.Replace(version, " ", "-", -1)
 	relTag = strings.Replace(relTag, ":", "-", -1)
 	relTag = strings.Replace(relTag, ",", "", -1)
-	return relPrefix + "." + relTag
+	relTag = relPrefix + "." + relTag
+
+	if relSuffix != "" {
+		relTag += "." + relSuffix
+	}
+
+	return relTag
 }
 
 // commitID returns the abbreviated commit-id hash of the last commit.
@@ -68,5 +79,12 @@ func commitID() string {
 }
 
 func main() {
-	fmt.Println(genLDFlags(time.Now().UTC().Format(time.RFC3339)))
+	var version string
+	if len(os.Args) > 1 {
+		version = os.Args[1]
+	} else {
+		version = time.Now().UTC().Format(time.RFC3339)
+	}
+
+	fmt.Println(genLDFlags(version))
 }

buildscripts/race.sh

@@ -3,5 +3,5 @@
 set -e
 
 for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -race --timeout 50m "$d"
+    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done

buildscripts/verify-build.sh

@@ -56,40 +56,41 @@ function start_minio_erasure()
 
 function start_minio_erasure_sets()
 {
-    "${MINIO[@]}" server "${WORK_DIR}/erasure-disk-sets{1...32}" >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
+    export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
+    "${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 &
     sleep 15
 }
 
-function start_minio_zone_erasure_sets()
+function start_minio_pool_erasure_sets()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
-
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
 
     sleep 40
 }
 
-function start_minio_zone_erasure_sets_ipv6()
+function start_minio_pool_erasure_sets_ipv6()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
-
-    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9001.log" 2>&1 &
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 
     sleep 40
 }
 
 function start_minio_dist_erasure()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
+    for i in $(seq 0 3); do
+        "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
+    done
 
     sleep 40
 }
@@ -112,7 +113,8 @@ function run_test_fs()
     return "$rv"
 }
 
-function run_test_erasure_sets() {
+function run_test_erasure_sets()
+{
     start_minio_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
@@ -129,9 +131,9 @@ function run_test_erasure_sets() {
     return "$rv"
 }
 
-function run_test_zone_erasure_sets()
+function run_test_pool_erasure_sets()
 {
-    start_minio_zone_erasure_sets
+    start_minio_pool_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
@@ -142,20 +144,20 @@ function run_test_zone_erasure_sets()
     if [ "$rv" -ne 0 ]; then
         for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/zone-minio-900$i.log"
+            cat "$WORK_DIR/pool-minio-900$i.log"
         done
     fi
 
     for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/zone-minio-900$i.log"
+        rm -f "$WORK_DIR/pool-minio-900$i.log"
     done
 
     return "$rv"
 }
 
-function run_test_zone_erasure_sets_ipv6()
+function run_test_pool_erasure_sets_ipv6()
 {
-    start_minio_zone_erasure_sets_ipv6
+    start_minio_pool_erasure_sets_ipv6
 
     export SERVER_ENDPOINT="[::1]:9000"
 
@@ -168,12 +170,12 @@ function run_test_zone_erasure_sets_ipv6()
     if [ "$rv" -ne 0 ]; then
         for i in $(seq 0 1); do
             echo "server$i log:"
-            cat "$WORK_DIR/zone-minio-ipv6-900$i.log"
+            cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
         done
     fi
 
     for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/zone-minio-ipv6-900$i.log"
+        rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
     done
 
     return "$rv"
@@ -220,7 +222,7 @@ function run_test_dist_erasure()
 
     rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
 
-   return "$rv"
+    return "$rv"
 }
 
 function purge()
@@ -293,14 +295,14 @@ function main()
     fi
 
     echo "Testing in Distributed Eraure expanded setup"
-    if ! run_test_zone_erasure_sets; then
+    if ! run_test_pool_erasure_sets; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1
     fi
 
     echo "Testing in Distributed Erasure expanded setup with ipv6"
-    if ! run_test_zone_erasure_sets_ipv6; then
+    if ! run_test_pool_erasure_sets_ipv6; then
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1

buildscripts/verify-healing.sh

@@ -29,44 +29,52 @@ MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
 
 function start_minio_3_node() {
-    declare -a minio_pids
-    declare -a ARGS
-    export MINIO_ACCESS_KEY=minio
-    export MINIO_SECRET_KEY=minio123
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
     export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
     start_port=$(shuf -i 10000-65000 -n 1)
+    args=""
     for i in $(seq 1 3); do
-        ARGS+=("http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/")
+        args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
-    "${MINIO[@]}" --address ":$[$start_port+1]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    minio_pids[0]=$!
-    disown "${minio_pids[0]}"
+    "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+    disown $!
 
-    "${MINIO[@]}" --address ":$[$start_port+2]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    minio_pids[1]=$!
-    disown "${minio_pids[1]}"
+    "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+    disown $!
 
-    "${MINIO[@]}" --address ":$[$start_port+3]" ${ARGS[@]} > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    minio_pids[2]=$!
-    disown "${minio_pids[2]}"
+    "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+    disown $!
 
     sleep "$1"
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            for i in $(seq 1 3); do
-                echo "server$i log:"
-                cat "${WORK_DIR}/dist-minio-server$i.log"
-            done
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
+    if [ "$(pgrep -c minio)" -ne 3 ]; then
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+    if ! pkill minio; then
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-server$i.log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+
+    sleep 1;
+    if pgrep minio; then
         # forcibly killing, to proceed further properly.
-        kill -9 "$pid"
-        sleep 1 # wait 1sec per pid
-    done
+        if ! pkill -9 minio; then
+            echo "no minio process running anymore, proceed."
+        fi
+    fi
 }
 
 

cmd/admin-bucket-handlers.go

@@ -23,9 +23,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -54,12 +52,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Turn off quota commands if data usage info is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL)
-		return
-	}
-
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -194,7 +186,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-
 	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/admin-handlers-config-kv.go

@@ -49,7 +49,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
+	cred, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return cred, nil
@@ -130,8 +130,8 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	dynamic, err := cfg.ReadConfig(bytes.NewReader(kvBytes))
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -158,6 +158,17 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
+	// Apply dynamic values.
+	if err := applyDynamicConfig(GlobalContext, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	globalNotificationSys.SignalService(serviceReloadDynamic)
+
+	// If all values were dynamic, tell the client.
+	if dynamic {
+		w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
+	}
 	writeSuccessResponseHeadersOnly(w)
 }
 
@@ -266,7 +277,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -378,7 +389,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	cfg := newServerConfig()
-	if _, err = cfg.ReadFrom(bytes.NewReader(kvBytes)); err != nil {
+	if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"path"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -42,7 +43,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return nil, cred
@@ -358,7 +359,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
+	accessKey := path.Clean(vars["accessKey"])
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -373,23 +374,29 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if cred.IsTemp() || cred.IsServiceAccount() {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccountNotEligible), r.URL)
-		return
-	}
-
 	// Not allowed to add a user with same access key as root credential
 	if owner && accessKey == cred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
 		return
 	}
 
+	if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
+		// Incoming access key matches parent user then we should
+		// reject password change requests.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
 	implicitPerm := accessKey == cred.AccessKey
 	if !implicitPerm {
+		parentUser := cred.ParentUser
+		if parentUser == "" {
+			parentUser = cred.AccessKey
+		}
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     accessKey,
+			AccountName:     parentUser,
 			Action:          iampolicy.CreateUserAdminAction,
-			ConditionValues: getConditionValues(r, "", accessKey, claims),
+			ConditionValues: getConditionValues(r, "", parentUser, claims),
 			IsOwner:         owner,
 			Claims:          claims,
 		}) {

cmd/admin-handlers.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bandwidth"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -204,9 +205,10 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	var objectAPI ObjectLayer
-	if serviceSig == serviceRestart {
+	switch serviceSig {
+	case serviceRestart:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
-	} else {
+	case serviceStop:
 		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
 	}
 	if objectAPI == nil {
@@ -290,7 +292,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	// ignores any errors here.
-	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
 
 	// Collect any disk healing.
 	healing, _ := getAggregatedBackgroundHealState(ctx)
@@ -371,14 +373,12 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 		if peerLock == nil {
 			continue
 		}
-		for _, locks := range peerLock.Locks {
-			for k, v := range locks {
-				for _, lockReqInfo := range v {
-					if val, ok := entryMap[lockReqInfo.UID]; ok {
-						val.ServerList = append(val.ServerList, peerLock.Addr)
-					} else {
-						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
-					}
+		for k, v := range peerLock.Locks {
+			for _, lockReqInfo := range v {
+				if val, ok := entryMap[lockReqInfo.UID]; ok {
+					val.ServerList = append(val.ServerList, peerLock.Addr)
+				} else {
+					entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 				}
 			}
 		}
@@ -400,7 +400,7 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 // PeerLocks holds server information result of one node
 type PeerLocks struct {
 	Addr  string
-	Locks GetLocksResp
+	Locks map[string][]lockRequesterInfo
 }
 
 // TopLocksHandler Get list of locks in use
@@ -461,8 +461,15 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 
 	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
-	if objectAPI == nil {
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -559,8 +566,15 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 
 	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
-	if objectAPI == nil {
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -900,7 +914,7 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	}
 
 	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
+	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return nil, cred
@@ -1032,7 +1046,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1398,7 +1412,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	}()
 
-	ticker := time.NewTicker(30 * time.Second)
+	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 
 	for {
@@ -1428,23 +1442,51 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "BandwidthMonitor")
 
+	defer logger.AuditLog(w, r, "BandwidthMonitor", mustGetClaimsFromToken(r))
+
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.BandwidthMonitorAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
 
 	setEventStreamHeaders(w)
+	reportCh := make(chan bandwidth.Report, 1)
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
 	bucketsRequestedString := r.URL.Query().Get("buckets")
 	bucketsRequested := strings.Split(bucketsRequestedString, ",")
-	consolidatedReport := globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
-	enc := json.NewEncoder(w)
-	err := enc.Encode(consolidatedReport)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+	go func() {
+		for {
+			reportCh <- globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				time.Sleep(2 * time.Second)
+			}
+		}
+	}()
+	for {
+		select {
+		case report := <-reportCh:
+			enc := json.NewEncoder(w)
+			err := enc.Encode(report)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-keepAliveTicker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
 	}
-	w.(http.Flusher).Flush()
 }
 
 // ServerInfoHandler - GET /minio/admin/v3/info
@@ -1455,22 +1497,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
-	if objectAPI == nil {
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
 
-	buckets := madmin.Buckets{}
-	objects := madmin.Objects{}
-	usage := madmin.Usage{}
-
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err == nil {
-		buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
-		objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
-		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
-	}
-
 	vault := fetchVaultStatus()
 
 	ldap := madmin.LDAP{}
@@ -1492,31 +1525,54 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
 
-	// Fetching the Storage information, ignore any errors.
-	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
-
-	var backend interface{}
-	if storageInfo.Backend.Type == BackendType(madmin.Erasure) {
-		backend = madmin.ErasureBackend{
-			Type:             madmin.ErasureType,
-			OnlineDisks:      storageInfo.Backend.OnlineDisks.Sum(),
-			OfflineDisks:     storageInfo.Backend.OfflineDisks.Sum(),
-			StandardSCData:   storageInfo.Backend.StandardSCData,
-			StandardSCParity: storageInfo.Backend.StandardSCParity,
-			RRSCData:         storageInfo.Backend.RRSCData,
-			RRSCParity:       storageInfo.Backend.RRSCParity,
-		}
-	} else {
-		backend = madmin.FSBackend{
-			Type: madmin.FsType,
-		}
-	}
-
-	mode := "online"
 	server := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
 	servers = append(servers, server)
 
+	var backend interface{}
+	mode := madmin.ObjectLayerInitializing
+
+	buckets := madmin.Buckets{}
+	objects := madmin.Objects{}
+	usage := madmin.Usage{}
+
+	objectAPI := newObjectLayerFn()
+	if objectAPI != nil {
+		mode = madmin.ObjectLayerOnline
+		// Load data usage
+		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+		if err == nil {
+			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
+			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
+			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
+		}
+
+		// Fetching the backend information
+		backendInfo := objectAPI.BackendInfo()
+		if backendInfo.Type == BackendType(madmin.Erasure) {
+			// Calculate the number of online/offline disks of all nodes
+			var allDisks []madmin.Disk
+			for _, s := range servers {
+				allDisks = append(allDisks, s.Disks...)
+			}
+			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+
+			backend = madmin.ErasureBackend{
+				Type:             madmin.ErasureType,
+				OnlineDisks:      onlineDisks.Sum(),
+				OfflineDisks:     offlineDisks.Sum(),
+				StandardSCData:   backendInfo.StandardSCData,
+				StandardSCParity: backendInfo.StandardSCParity,
+				RRSCData:         backendInfo.RRSCData,
+				RRSCParity:       backendInfo.RRSCParity,
+			}
+		} else {
+			backend = madmin.FSBackend{
+				Type: madmin.FsType,
+			}
+		}
+	}
+
 	domain := globalDomainNames
 	services := madmin.Services{
 		Vault:         vault,
@@ -1526,25 +1582,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
-	// find all disks which belong to each respective endpoints
-	for i := range servers {
-		for _, disk := range storageInfo.Disks {
-			if strings.Contains(disk.Endpoint, servers[i].Endpoint) {
-				servers[i].Disks = append(servers[i].Disks, disk)
-			}
-		}
-	}
-
-	// add all the disks local to this server.
-	for _, disk := range storageInfo.Disks {
-		if disk.DrivePath == "" && disk.Endpoint == "" {
-			continue
-		}
-		if disk.Endpoint == disk.DrivePath {
-			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
-		}
-	}
-
 	infoMsg := madmin.InfoMessage{
 		Mode:         mode,
 		Domain:       domain,

cmd/admin-handlers_test.go

@@ -364,7 +364,7 @@ func TestExtractHealInitParams(t *testing.T) {
 	// Test all combinations!
 	for pIdx, parms := range qParmsArr {
 		for vIdx, vars := range varsArr {
-			_, err := extractHealInitParams(vars, parms, bytes.NewBuffer([]byte(body)))
+			_, err := extractHealInitParams(vars, parms, bytes.NewReader([]byte(body)))
 			isErrCase := false
 			if pIdx < 4 || vIdx < 1 {
 				isErrCase = true

cmd/admin-heal-ops.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -93,15 +95,15 @@ type allHealState struct {
 }
 
 // newHealState - initialize global heal state management
-func newHealState() *allHealState {
-	healState := &allHealState{
+func newHealState(cleanup bool) *allHealState {
+	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
 		healLocalDisks: map[Endpoint]struct{}{},
 	}
-
-	go healState.periodicHealSeqsClean(GlobalContext)
-
-	return healState
+	if cleanup {
+		go hstate.periodicHealSeqsClean(GlobalContext)
+	}
+	return hstate
 }
 
 func (ahs *allHealState) healDriveCount() int {
@@ -143,9 +145,13 @@ func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
+	periodicTimer := time.NewTimer(time.Minute * 5)
+	defer periodicTimer.Stop()
+
 	for {
 		select {
-		case <-time.After(time.Minute * 5):
+		case <-periodicTimer.C:
+			periodicTimer.Reset(time.Minute * 5)
 			now := UTCNow()
 			ahs.Lock()
 			for path, h := range ahs.healSeqMap {
@@ -657,7 +663,9 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
+	globalHealConfigMu.Lock()
 	opts := globalHealConfig
+	globalHealConfigMu.Unlock()
 
 	// Send heal request
 	task := healTask{
@@ -669,10 +677,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
-	} else {
-		if opts.Bitrot {
-			task.opts.ScanMode = madmin.HealDeepScan
-		}
+	}
+	if opts.Bitrot {
+		task.opts.ScanMode = madmin.HealDeepScan
 	}
 
 	// Wait and proceed if there are active requests
@@ -776,13 +783,18 @@ func (h *healSequence) healFromSourceCh() {
 }
 
 func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
-	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(objAPI, minioConfigPrefix)(); err != nil {
-		return err
+	// Try to pro-actively heal backend-encrypted file.
+	if err := h.queueHealTask(healSource{
+		bucket: minioMetaBucket,
+		object: backendEncryptedFile,
+	}, madmin.HealItemBucketMetadata); err != nil {
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
 	}
 
-	// Start healing the bucket config prefix.
-	return h.healMinioSysMeta(objAPI, bucketConfigPrefix)()
+	// Start healing the config prefix.
+	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
 }
 
 func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
@@ -819,6 +831,11 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 				return errHealStopSignalled
 			}
 
+			// Skip metacache entries healing
+			if strings.HasPrefix(object, "buckets/.minio.sys/.metacache/") {
+				return nil
+			}
+
 			err := h.queueHealTask(healSource{
 				bucket:    bucket,
 				object:    object,
@@ -855,11 +872,16 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	buckets, err := objAPI.ListBucketsHeal(h.ctx)
+	buckets, err := objAPI.ListBuckets(h.ctx)
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
+	// Heal latest buckets first.
+	sort.Slice(buckets, func(i, j int) bool {
+		return buckets[i].Created.After(buckets[j].Created)
+	})
+
 	for _, bucket := range buckets {
 		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
 			return err

cmd/admin-router.go

@@ -20,8 +20,6 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -172,27 +170,25 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		}
 
 		if globalIsDistErasure || globalIsErasure {
-			// Quota operations
-			if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-				// GetBucketQuotaConfig
-				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-					httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-				// PutBucketQuotaConfig
-				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+			// GetBucketQuotaConfig
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
+				httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+			// PutBucketQuotaConfig
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
+				httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
 
-				// Bucket replication operations
-				// GetBucketTargetHandler
-				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-					httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-				// SetRemoteTargetHandler
-				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-					httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
-				// RemoveRemoteTargetHandler
-				adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-					httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
-			}
+			// Bucket replication operations
+			// GetBucketTargetHandler
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+			// SetRemoteTargetHandler
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+			// RemoveRemoteTargetHandler
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		}
+
 		// -- Top APIs --
 		// Top locks
 		if globalIsDistErasure {

cmd/admin-server-info.go

@@ -25,6 +25,7 @@ import (
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
 func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
+	var localEndpoints Endpoints
 	addr := r.Host
 	if globalIsDistErasure {
 		addr = GetLocalPeer(endpointServerPools)
@@ -39,11 +40,12 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			if endpoint.IsLocal {
 				// Only proceed for local endpoints
 				network[nodeName] = "online"
+				localEndpoints = append(localEndpoints, endpoint)
 				continue
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := IsServerResolvable(endpoint); err == nil {
+				if err := isServerResolvable(endpoint); err == nil {
 					network[nodeName] = "online"
 				} else {
 					network[nodeName] = "offline"
@@ -52,6 +54,11 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		}
 	}
 
+	localDisks, _ := initStorageDisksWithErrors(localEndpoints)
+	defer closeStorageDisks(localDisks)
+
+	storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings())
+
 	return madmin.ServerProperties{
 		State:    "ok",
 		Endpoint: addr,
@@ -59,5 +66,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		Version:  Version,
 		CommitID: CommitID,
 		Network:  network,
+		Disks:    storageInfo.Disks,
 	}
 }

cmd/api-errors.go

@@ -120,7 +120,6 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
-	ErrBucketReplicationDisabledError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -263,7 +262,6 @@ const (
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
 	ErrAdminNoSuchQuotaConfiguration
-	ErrAdminBucketQuotaDisabled
 
 	ErrHealNotImplemented
 	ErrHealNoSuchProcess
@@ -889,11 +887,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Versioning must be 'Enabled' on the bucket to add a replication target",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrBucketReplicationDisabledError: {
-		Code:           "XMinioAdminBucketReplicationDisabled",
-		Description:    "Replication specified but disk usage crawl is disabled on MinIO server",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",
@@ -1215,11 +1208,6 @@ var errorCodes = errorCodeMap{
 		Description:    "The quota configuration does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrAdminBucketQuotaDisabled: {
-		Code:           "XMinioAdminBucketQuotaDisabled",
-		Description:    "Quota specified but disk usage crawl is disabled on MinIO server",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrInsecureClientRequest: {
 		Code:           "XMinioInsecureClientRequest",
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",

cmd/api-headers.go

@@ -157,19 +157,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	if opts.PartNumber > 0 {
-		var start, end int64
-		for i := 0; i < len(objInfo.Parts) && i < opts.PartNumber; i++ {
-			start = end
-			end = start + objInfo.Parts[i].ActualSize - 1
-		}
-		rs = &HTTPRangeSpec{Start: start, End: end}
-		rangeLen = end - start + 1
-	} else {
-		// for providing ranged content
-		start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
-		if err != nil {
-			return err
-		}
+		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
+	}
+
+	// For providing ranged content
+	start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
+	if err != nil {
+		return err
 	}
 
 	// Set content length.

cmd/api-response.go

@@ -388,7 +388,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 	}
 	proto := handlers.GetSourceScheme(r)
 	if proto == "" {
-		proto = getURLScheme(globalIsSSL)
+		proto = getURLScheme(globalIsTLS)
 	}
 	u := &url.URL{
 		Host:   r.Host,

cmd/api-router.go

@@ -27,23 +27,35 @@ import (
 )
 
 func newHTTPServerFn() *xhttp.Server {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
 	return globalHTTPServer
 }
 
-func newObjectLayerFn() ObjectLayer {
+func setHTTPServer(h *xhttp.Server) {
 	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
+	globalHTTPServer = h
+	globalObjLayerMutex.Unlock()
+}
+
+func newObjectLayerFn() ObjectLayer {
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
 	return globalObjectAPI
 }
 
 func newCachedObjectLayerFn() CacheObjectLayer {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
+	globalObjLayerMutex.RLock()
+	defer globalObjLayerMutex.RUnlock()
 	return globalCacheObjectAPI
 }
 
+func setCacheObjectLayer(c CacheObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalCacheObjectAPI = c
+	globalObjLayerMutex.Unlock()
+}
+
 func setObjectLayer(o ObjectLayer) {
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = o
@@ -108,224 +120,225 @@ func registerAPIRouter(router *mux.Router) {
 		// Object operations
 		// HeadObject
 		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
+			collectAPIStats("headobject", maxClients(httpTraceAll(api.HeadObjectHandler))))
 		// CopyObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).
+			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(httpTraceAll(api.CopyObjectPartHandler)))).
 			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			collectAPIStats("putobjectpart", maxClients(httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("listobjectparts", maxClients(httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("completemutipartupload", maxClients(httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
+			collectAPIStats("newmultipartupload", maxClients(httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
 		// AbortMultipartUpload
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("abortmultipartupload", maxClients(httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
+			collectAPIStats("getobjectacl", maxClients(httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
+			collectAPIStats("putobjectacl", maxClients(httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
 		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("getobjecttagging", maxClients(httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
 		// PutObjectTagging
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("putobjecttagging", maxClients(httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
 		// DeleteObjectTagging
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("deleteobjecttagging", maxClients(httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
+			collectAPIStats("selectobjectcontent", maxClients(httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
+			collectAPIStats("getobjectretention", maxClients(httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
 		// GetObjectLegalHold
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
+			collectAPIStats("getobjectlegalhold", maxClients(httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
 		// GetObject
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
+			collectAPIStats("getobject", maxClients(httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
+			collectAPIStats("copyobject", maxClients(httpTraceAll(api.CopyObjectHandler))))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
+			collectAPIStats("putobjectretention", maxClients(httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
 		// PutObjectLegalHold
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
+			collectAPIStats("putobjectlegalhold", maxClients(httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
 
 		// PutObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler))))
+			collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectHandler))))
 		// DeleteObject
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler))))
+			collectAPIStats("deleteobject", maxClients(httpTraceAll(api.DeleteObjectHandler))))
 
 		/// Bucket operations
 		// GetBucketLocation
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
+			collectAPIStats("getbucketlocation", maxClients(httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
 		// GetBucketPolicy
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("getbucketpolicy", maxClients(httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
 		// GetBucketLifecycle
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// GetBucketEncryption
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("getbucketencryption", maxClients(httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
 		// GetBucketObjectLockConfig
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+			collectAPIStats("getbucketobjectlockconfiguration", maxClients(httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
 		// GetBucketReplicationConfig
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketreplicationconfiguration", httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("getbucketreplicationconfiguration", maxClients(httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
 
 		// GetBucketVersioning
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
+			collectAPIStats("getbucketversioning", maxClients(httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
 		// GetBucketNotification
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
+			collectAPIStats("getbucketnotification", maxClients(httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
 		// ListenNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
+			collectAPIStats("getbucketacl", maxClients(httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
+			collectAPIStats("putbucketacl", maxClients(httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
+			collectAPIStats("getbucketcors", maxClients(httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
+			collectAPIStats("getbucketwebsite", maxClients(httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
+			collectAPIStats("getbucketaccelerate", maxClients(httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
+			collectAPIStats("getbucketrequestpayment", maxClients(httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
+			collectAPIStats("getbucketlogging", maxClients(httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// GetBucketTaggingHandler
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("getbuckettagging", maxClients(httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
+			collectAPIStats("deletebucketwebsite", maxClients(httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
 		// DeleteBucketTaggingHandler
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("deletebuckettagging", maxClients(httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
 
 		// ListMultipartUploads
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
+			collectAPIStats("listmultipartuploads", maxClients(httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
 		// ListObjectsV2M
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
+			collectAPIStats("listobjectsv2M", maxClients(httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
+			collectAPIStats("listobjectsv2", maxClients(httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
 		// ListObjectVersions
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectversions", httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
+			collectAPIStats("listobjectversions", maxClients(httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler))))
+			collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("putbucketlifecycle", maxClients(httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// PutBucketReplicationConfig
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketreplicationconfiguration", httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("putbucketreplicationconfiguration", maxClients(httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
 		// GetObjectRetention
 
 		// PutBucketEncryption
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("putbucketencryption", maxClients(httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
 
 		// PutBucketPolicy
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("putbucketpolicy", maxClients(httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
 
 		// PutBucketObjectLockConfig
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+			collectAPIStats("putbucketobjectlockconfig", maxClients(httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
 		// PutBucketTaggingHandler
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbuckettagging", httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("putbuckettagging", maxClients(httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
 		// PutBucketVersioning
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
+			collectAPIStats("putbucketversioning", maxClients(httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
 		// PutBucketNotification
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
+			collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
 		// PutBucket
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler))))
+			collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler))))
 		// HeadBucket
 		bucket.Methods(http.MethodHead).HandlerFunc(
-			maxClients(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler))))
+			collectAPIStats("headbucket", maxClients(httpTraceAll(api.HeadBucketHandler))))
 		// PostPolicy
 		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			maxClients(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler))))
+			collectAPIStats("postpolicybucket", maxClients(httpTraceHdrs(api.PostPolicyBucketHandler))))
 		// DeleteMultipleObjects
 		bucket.Methods(http.MethodPost).HandlerFunc(
-			maxClients(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
+			collectAPIStats("deletemultipleobjects", maxClients(httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
 		// DeleteBucketPolicy
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("deletebucketpolicy", maxClients(httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
 		// DeleteBucketReplication
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketreplicationconfiguration", httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("deletebucketreplicationconfiguration", maxClients(httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
 		// DeleteBucketLifecycle
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("deletebucketlifecycle", maxClients(httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// DeleteBucketEncryption
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("deletebucketencryption", maxClients(httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler))))
+			collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler))))
 		// PostRestoreObject
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("restoreobject", httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
+			collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
 	}
 
 	/// Root operation
 
 	// ListenNotification
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
+		collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
 
 	// ListBuckets
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
+		collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
 
 	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
 	// than failing with UnknownAPIRequest we simply handle it for now.
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
-		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
+		collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))

cmd/auth-handler.go

@@ -151,9 +151,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	return cred, claims, owner, ErrNone
 }
 
-// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
-// It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+// checkAdminRequestAuth checks for authentication and authorization for the incoming
+// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
+// are automatically rejected.
+func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
 	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
@@ -467,16 +468,6 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	return ErrNone
 }
 
-// authHandler - handles all the incoming authorization headers and validates them if possible.
-type authHandler struct {
-	handler http.Handler
-}
-
-// setAuthHandler to validate authorization header for the incoming request.
-func setAuthHandler(h http.Handler) http.Handler {
-	return authHandler{h}
-}
-
 // List of all support S3 auth types.
 var supportedS3AuthTypes = map[authType]struct{}{
 	authTypeAnonymous:       {},
@@ -494,26 +485,29 @@ func isSupportedS3AuthType(aType authType) bool {
 	return ok
 }
 
-// handler for validating incoming authorization headers.
-func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	aType := getRequestAuthType(r)
-	if isSupportedS3AuthType(aType) {
-		// Let top level caller validate for anonymous and known signed requests.
-		a.handler.ServeHTTP(w, r)
-		return
-	} else if aType == authTypeJWT {
-		// Validate Authorization header if its valid for JWT request.
-		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
-			w.WriteHeader(http.StatusUnauthorized)
+// setAuthHandler to validate authorization header for the incoming request.
+func setAuthHandler(h http.Handler) http.Handler {
+	// handler for validating incoming authorization headers.
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		aType := getRequestAuthType(r)
+		if isSupportedS3AuthType(aType) {
+			// Let top level caller validate for anonymous and known signed requests.
+			h.ServeHTTP(w, r)
+			return
+		} else if aType == authTypeJWT {
+			// Validate Authorization header if its valid for JWT request.
+			if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			h.ServeHTTP(w, r)
+			return
+		} else if aType == authTypeSTS {
+			h.ServeHTTP(w, r)
 			return
 		}
-		a.handler.ServeHTTP(w, r)
-		return
-	} else if aType == authTypeSTS {
-		a.handler.ServeHTTP(w, r)
-		return
-	}
-	writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
+	})
 }
 
 func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, map[string]interface{}, APIErrorCode) {

cmd/auth-handler_test.go

@@ -421,7 +421,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuthType(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/background-heal-ops.go

@@ -54,20 +54,25 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
-func waitForLowHTTPReq(tolerance int, maxWait time.Duration) {
+func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
+	// No need to wait run at full speed.
+	if maxIO <= 0 {
+		return
+	}
+
 	// At max 10 attempts to wait with 100 millisecond interval before proceeding
 	waitCount := 10
 	waitTick := 100 * time.Millisecond
 
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	toleranceFn := func() int {
-		return tolerance + globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers()
+	maxIOFn := func() int {
+		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalHTTPTrace.NumSubscribers())
 	}
 
 	if httpServer := newHTTPServerFn(); httpServer != nil {
 		// Any requests in progress, delay the heal.
-		for httpServer.GetRequestCount() >= toleranceFn() {
+		for httpServer.GetRequestCount() >= maxIOFn() {
 			time.Sleep(waitTick)
 			waitCount--
 			if waitCount == 0 {
@@ -97,7 +102,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			case task.bucket == SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
 			case task.bucket != "" && task.object == "":
-				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts.DryRun, task.opts.Remove)
+				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
 			case task.bucket != "" && task.object != "":
 				res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
 			}

cmd/background-newdisks-heal-ops.go

@@ -20,10 +20,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/console"
 )
 
 const (
@@ -46,16 +48,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	var bgSeq *healSequence
-	var found bool
-
-	for {
-		bgSeq, found = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-		if found {
-			break
-		}
-		time.Sleep(time.Second)
-	}
+	bgSeq := mustGetHealSequence(ctx)
 
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
@@ -118,12 +111,17 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 //  2. Only the node hosting the disk is responsible to perform the heal
 func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
+	diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval)
+	defer diskCheckTimer.Stop()
 wait:
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.After(defaultMonitorNewDiskInterval):
+		case <-diskCheckTimer.C:
+			// Reset to next interval.
+			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
+
 			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
 
 			healDisks := globalBackgroundHealState.getHealLocalDisks()
@@ -143,6 +141,10 @@ wait:
 				}
 			}
 
+			if serverDebugLog {
+				console.Debugf("disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+			}
+
 			// heal only if new disks found.
 			for _, endpoint := range healDisks {
 				disk, format, err := connectEndpoint(endpoint)
@@ -151,33 +153,39 @@ wait:
 					continue
 				}
 
-				zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
-				if zoneIdx < 0 {
+				poolIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
+				if poolIdx < 0 {
 					continue
 				}
 
 				// Calculate the set index where the current endpoint belongs
-				z.serverPools[zoneIdx].erasureDisksMu.RLock()
+				z.serverPools[poolIdx].erasureDisksMu.RLock()
 				// Protect reading reference format.
-				setIndex, _, err := findDiskIndex(z.serverPools[zoneIdx].format, format)
-				z.serverPools[zoneIdx].erasureDisksMu.RUnlock()
+				setIndex, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
+				z.serverPools[poolIdx].erasureDisksMu.RUnlock()
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					continue
 				}
 
-				erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
+				erasureSetInZoneDisksToHeal[poolIdx][setIndex] = append(erasureSetInZoneDisksToHeal[poolIdx][setIndex], disk)
 			}
 
-			buckets, _ := z.ListBucketsHeal(ctx)
+			buckets, _ := z.ListBuckets(ctx)
+
+			// Heal latest buckets first.
+			sort.Slice(buckets, func(i, j int) bool {
+				return buckets[i].Created.After(buckets[j].Created)
+			})
+
 			for i, setMap := range erasureSetInZoneDisksToHeal {
 				for setIndex, disks := range setMap {
 					for _, disk := range disks {
-						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+						logger.Info("Healing disk '%s' on %s pool", disk, humanize.Ordinal(i+1))
 
 						// So someone changed the drives underneath, healing tracker missing.
 						if !disk.Healing() {
-							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s zone", disk, humanize.Ordinal(i+1))
+							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
 							diskID, err := disk.GetDiskID()
 							if err != nil {
 								logger.LogIf(ctx, err)
@@ -201,7 +209,7 @@ wait:
 							continue
 						}
 
-						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
+						logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
 
 						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
 							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {

cmd/benchmark-utils_test.go

@@ -55,7 +55,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -114,7 +114,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			md5hex := getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -200,7 +200,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -301,7 +301,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -340,7 +340,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/bitrot-streaming.go

@@ -91,7 +91,8 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 // ReadAt() implementation which verifies the bitrot hash available as part of the stream.
 type streamingBitrotReader struct {
 	disk       StorageAPI
-	rc         io.ReadCloser
+	data       []byte
+	rc         io.Reader
 	volume     string
 	filePath   string
 	tillOffset int64
@@ -105,7 +106,10 @@ func (b *streamingBitrotReader) Close() error {
 	if b.rc == nil {
 		return nil
 	}
-	return b.rc.Close()
+	if closer, ok := b.rc.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
 }
 
 func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
@@ -119,11 +123,16 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
-		b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		if len(b.data) == 0 {
+			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		} else {
+			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
+		}
 		if err != nil {
 			return 0, err
 		}
 	}
+
 	if offset != b.currOffset {
 		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
@@ -150,10 +159,11 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 }
 
 // Returns streaming bitrot reader implementation.
-func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
+func newStreamingBitrotReader(disk StorageAPI, data []byte, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
 	h := algo.New()
 	return &streamingBitrotReader{
 		disk,
+		data,
 		nil,
 		volume,
 		filePath,

cmd/bitrot.go

@@ -103,9 +103,9 @@ func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, alg
 	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }
 
-func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
+func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
 	if algo == HighwayHash256S {
-		return newStreamingBitrotReader(disk, bucket, filePath, tillOffset, algo, shardSize)
+		return newStreamingBitrotReader(disk, data, bucket, filePath, tillOffset, algo, shardSize)
 	}
 	return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
 }

cmd/bitrot_test.go

@@ -62,7 +62,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	}
 	writer.(io.Closer).Close()
 
-	reader := newBitrotReader(disk, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
+	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
 	if _, err = reader.ReadAt(b, 0); err != nil {
 		log.Fatal(err)

cmd/bucket-handlers.go

@@ -28,11 +28,11 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -41,7 +41,6 @@ import (
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
@@ -433,6 +432,20 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 			continue
 		}
+		if object.VersionID != "" && object.VersionID != nullVersionID {
+			if _, err := uuid.Parse(object.VersionID); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("invalid version-id specified %w", err))
+				apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion)
+				dErrs[index] = DeleteError{
+					Code:      apiErr.Code,
+					Message:   apiErr.Description,
+					Key:       object.ObjectName,
+					VersionID: object.VersionID,
+				}
+				continue
+			}
+		}
+
 		if replicateDeletes || hasLockEnabled || hasLifecycleConfig {
 			goi, gerr = getObjectInfoFn(ctx, bucket, object.ObjectName, ObjectOptions{
 				VersionID: object.VersionID,
@@ -503,7 +516,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
 			PurgeTransitioned:             dObjects[i].PurgeTransitioned,
 		}]
-		if errs[i] == nil || isErrObjectNotFound(errs[i]) {
+		if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
 			if replicateDeletes {
 				dObjects[i].DeleteMarkerReplicationStatus = deleteList[i].DeleteMarkerReplicationStatus
 				dObjects[i].VersionPurgeStatus = deleteList[i].VersionPurgeStatus
@@ -733,7 +746,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
+	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -880,7 +893,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
 		// This request header needs to be set prior to setting ObjectOptions
 		if !crypto.SSEC.IsRequested(r.Header) {
-			r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+			r.Header.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 		}
 	}
 
@@ -892,7 +905,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsRequested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
 				return
@@ -1321,11 +1334,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
-	// Turn off replication if disk crawl is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketReplicationDisabledError), r.URL)
-		return
-	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-handlers_test.go

@@ -35,7 +35,7 @@ func TestRemoveBucketHandler(t *testing.T) {
 
 func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewBuffer([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
+	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewReader([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
 	// if object upload fails stop the test.
 	if err != nil {
 		t.Fatalf("Error uploading object: <ERROR> %v", err)
@@ -669,7 +669,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
+		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)

cmd/bucket-lifecycle.go

@@ -28,7 +28,6 @@ import (
 
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	sse "github.com/minio/minio/pkg/bucket/encryption"
@@ -341,13 +340,14 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 		return err
 	}
 	oi := gr.ObjInfo
-
 	if oi.TransitionStatus == lifecycle.TransitionComplete {
+		gr.Close()
 		return nil
 	}
 
 	putOpts := putTransitionOpts(oi)
 	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, "", "", putOpts); err != nil {
+		gr.Close()
 		return err
 	}
 	gr.Close()
@@ -356,11 +356,23 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
 	opts.VersionID = oi.VersionID
 	opts.TransitionStatus = lifecycle.TransitionComplete
+	eventName := event.ObjectTransitionComplete
 
-	if _, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-		return err
+	_, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	if err != nil {
+		eventName = event.ObjectTransitionFailed
 	}
-	return nil
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: oi.Bucket,
+		Object: ObjectInfo{
+			Name:      oi.Name,
+			VersionID: opts.VersionID,
+		},
+		Host: "Internal: [ILM-Transition]",
+	})
+	return err
 }
 
 // getLifecycleTransitionTargetArn returns transition ARN for storage class specified in the config.
@@ -535,7 +547,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 		if r.OutputLocation.S3.Prefix == "" {
 			return fmt.Errorf("Prefix is a required parameter in OutputLocation")
 		}
-		if r.OutputLocation.S3.Encryption.EncryptionType != crypto.SSEAlgorithmAES256 {
+		if r.OutputLocation.S3.Encryption.EncryptionType != xhttp.AmzEncryptionAES {
 			return NotImplemented{}
 		}
 	}
@@ -561,7 +573,7 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 		}
 		meta[xhttp.AmzObjectTagging] = rreq.OutputLocation.S3.Tagging.String()
 		if rreq.OutputLocation.S3.Encryption.EncryptionType != "" {
-			meta[crypto.SSEHeader] = crypto.SSEAlgorithmAES256
+			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
 			Versioned:        globalBucketVersioningSys.Enabled(bucket),

cmd/bucket-metadata-sys.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -168,7 +169,10 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		}
 		meta.ReplicationConfigXML = configData
 	case bucketTargetsFile:
-		meta.BucketTargetsConfigJSON = configData
+		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{bucket: meta.Name, bucketTargetsFile: bucketTargetsFile})
+		if err != nil {
+			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+		}
 	default:
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
@@ -191,7 +195,6 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 // This function should only be used with
 // - GetBucketInfo
 // - ListBuckets
-// - ListBucketsHeal (only in case of erasure coding mode)
 // For all other bucket specific metadata, use the relevant
 // calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
@@ -440,6 +443,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
+			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
+				// Ensure heal opts for bucket metadata be deep healed all the time.
+				ScanMode: madmin.HealDeepScan,
+			})
 			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
 			if err != nil {
 				return err
@@ -470,6 +477,15 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob
 	}
 }
 
+// Reset the state of the BucketMetadataSys.
+func (sys *BucketMetadataSys) Reset() {
+	sys.Lock()
+	for k := range sys.metadataMap {
+		delete(sys.metadataMap, k)
+	}
+	sys.Unlock()
+}
+
 // NewBucketMetadataSys - creates new policy system.
 func NewBucketMetadataSys() *BucketMetadataSys {
 	return &BucketMetadataSys{

cmd/bucket-metadata.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/rand"
 	"encoding/binary"
 	"encoding/json"
 	"encoding/xml"
@@ -28,6 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -37,6 +39,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/sio"
 )
 
 const (
@@ -61,31 +64,33 @@ var (
 // bucketMetadataFormat refers to the format.
 // bucketMetadataVersion can be used to track a rolling upgrade of a field.
 type BucketMetadata struct {
-	Name                    string
-	Created                 time.Time
-	LockEnabled             bool // legacy not used anymore.
-	PolicyConfigJSON        []byte
-	NotificationConfigXML   []byte
-	LifecycleConfigXML      []byte
-	ObjectLockConfigXML     []byte
-	VersioningConfigXML     []byte
-	EncryptionConfigXML     []byte
-	TaggingConfigXML        []byte
-	QuotaConfigJSON         []byte
-	ReplicationConfigXML    []byte
-	BucketTargetsConfigJSON []byte
+	Name                        string
+	Created                     time.Time
+	LockEnabled                 bool // legacy not used anymore.
+	PolicyConfigJSON            []byte
+	NotificationConfigXML       []byte
+	LifecycleConfigXML          []byte
+	ObjectLockConfigXML         []byte
+	VersioningConfigXML         []byte
+	EncryptionConfigXML         []byte
+	TaggingConfigXML            []byte
+	QuotaConfigJSON             []byte
+	ReplicationConfigXML        []byte
+	BucketTargetsConfigJSON     []byte
+	BucketTargetsConfigMetaJSON []byte
 
 	// Unexported fields. Must be updated atomically.
-	policyConfig       *policy.Policy
-	notificationConfig *event.Config
-	lifecycleConfig    *lifecycle.Lifecycle
-	objectLockConfig   *objectlock.Config
-	versioningConfig   *versioning.Versioning
-	sseConfig          *bucketsse.BucketSSEConfig
-	taggingConfig      *tags.Tags
-	quotaConfig        *madmin.BucketQuota
-	replicationConfig  *replication.Config
-	bucketTargetConfig *madmin.BucketTargets
+	policyConfig           *policy.Policy
+	notificationConfig     *event.Config
+	lifecycleConfig        *lifecycle.Lifecycle
+	objectLockConfig       *objectlock.Config
+	versioningConfig       *versioning.Versioning
+	sseConfig              *bucketsse.BucketSSEConfig
+	taggingConfig          *tags.Tags
+	quotaConfig            *madmin.BucketQuota
+	replicationConfig      *replication.Config
+	bucketTargetConfig     *madmin.BucketTargets
+	bucketTargetConfigMeta map[string]string
 }
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
@@ -100,7 +105,8 @@ func newBucketMetadata(name string) BucketMetadata {
 		versioningConfig: &versioning.Versioning{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
-		bucketTargetConfig: &madmin.BucketTargets{},
+		bucketTargetConfig:     &madmin.BucketTargets{},
+		bucketTargetConfigMeta: make(map[string]string),
 	}
 }
 
@@ -140,16 +146,16 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
 	err := b.Load(ctx, objectAPI, b.Name)
-	if err == nil {
-		return b, b.convertLegacyConfigs(ctx, objectAPI)
-	}
-
-	if !errors.Is(err, errConfigNotFound) {
+	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
 
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
-	return b, b.convertLegacyConfigs(ctx, objectAPI)
+	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
+		return b, err
+	}
+	// migrate unencrypted remote targets
+	return b, b.migrateTargetConfig(ctx, objectAPI)
 }
 
 // parseAllConfigs will parse all configs and populate the private fields.
@@ -234,7 +240,8 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	}
 
 	if len(b.BucketTargetsConfigJSON) != 0 {
-		if err = json.Unmarshal(b.BucketTargetsConfigJSON, b.bucketTargetConfig); err != nil {
+		b.bucketTargetConfig, err = parseBucketTargetConfig(b.Name, b.BucketTargetsConfigJSON, b.BucketTargetsConfigMetaJSON)
+		if err != nil {
 			return err
 		}
 	} else {
@@ -354,6 +361,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 	if err != nil {
 		return err
 	}
+
 	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
 	return saveConfig(ctx, api, configFile, data)
 }
@@ -373,3 +381,76 @@ func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string)
 	}
 	return nil
 }
+
+// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
+func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
+	var err error
+	// early return if no targets or already encrypted
+	if len(b.BucketTargetsConfigJSON) == 0 || GlobalKMS == nil || len(b.BucketTargetsConfigMetaJSON) != 0 {
+		return nil
+	}
+
+	encBytes, metaBytes, err := encryptBucketMetadata(b.Name, b.BucketTargetsConfigJSON, crypto.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
+	if err != nil {
+		return err
+	}
+
+	b.BucketTargetsConfigJSON = encBytes
+	b.BucketTargetsConfigMetaJSON = metaBytes
+	return b.Save(ctx, objectAPI)
+}
+
+// encrypt bucket metadata if kms is configured.
+func encryptBucketMetadata(bucket string, input []byte, kmsContext crypto.Context) (output, metabytes []byte, err error) {
+	var sealedKey crypto.SealedKey
+	if GlobalKMS == nil {
+		output = input
+		return
+	}
+	var (
+		key    [32]byte
+		encKey []byte
+	)
+	metadata := make(map[string]string)
+	key, encKey, err = GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), kmsContext)
+	if err != nil {
+		return
+	}
+	outbuf := bytes.NewBuffer(nil)
+	objectKey := crypto.GenerateKey(key, rand.Reader)
+	sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
+	crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
+	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
+	if err != nil {
+		return output, metabytes, err
+	}
+	metabytes, err = json.Marshal(metadata)
+	if err != nil {
+		return
+	}
+	return outbuf.Bytes(), metabytes, nil
+}
+
+// decrypt bucket metadata if kms is configured.
+func decryptBucketMetadata(input []byte, bucket string, meta map[string]string, kmsContext crypto.Context) ([]byte, error) {
+	if GlobalKMS == nil {
+		return nil, errKMSNotConfigured
+	}
+	keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(meta)
+
+	if err != nil {
+		return nil, err
+	}
+	extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, kmsContext)
+	if err != nil {
+		return nil, err
+	}
+	var objectKey crypto.ObjectKey
+	if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, ""); err != nil {
+		return nil, err
+	}
+	outbuf := bytes.NewBuffer(nil)
+	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
+
+	return outbuf.Bytes(), err
+}

cmd/bucket-metadata_gen.go

@@ -102,6 +102,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
+		case "BucketTargetsConfigMetaJSON":
+			z.BucketTargetsConfigMetaJSON, err = dc.ReadBytes(z.BucketTargetsConfigMetaJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -115,9 +121,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 13
+	// map header, size 14
 	// write "Name"
-	err = en.Append(0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -246,15 +252,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 		return
 	}
+	// write "BucketTargetsConfigMetaJSON"
+	err = en.Append(0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.BucketTargetsConfigMetaJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 13
+	// map header, size 14
 	// string "Name"
-	o = append(o, 0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -292,6 +308,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "BucketTargetsConfigJSON"
 	o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
 	o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON)
+	// string "BucketTargetsConfigMetaJSON"
+	o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
 	return
 }
 
@@ -391,6 +410,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
+		case "BucketTargetsConfigMetaJSON":
+			z.BucketTargetsConfigMetaJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigMetaJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -405,6 +430,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON)
 	return
 }

cmd/bucket-object-lock.go

@@ -97,6 +97,9 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 				return ErrNone
 			}
 		}
+		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) {
+			return ErrNone
+		}
 		return toAPIErrorCode(ctx, gerr)
 	}
 

cmd/bucket-replication.go

@@ -166,7 +166,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 		Name:         dobj.ObjectName,
 		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
 		UserTags:     oi.UserTags,
-		DeleteMarker: true,
+		DeleteMarker: oi.DeleteMarker,
 		VersionID:    dobj.VersionID,
 	}
 	return oi.DeleteMarker, rcfg.Replicate(opts)
@@ -287,7 +287,7 @@ func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]
 	if oi.UserTags != "" {
 		meta[xhttp.AmzObjectTagging] = oi.UserTags
 	}
-	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339)
+	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
 	meta[xhttp.MinIOSourceETag] = oi.ETag
 	meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
 	return meta
@@ -330,7 +330,7 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 		putOpts.Mode = rmode
 	}
 	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
-		rdate, err := time.Parse(time.RFC3339, retainDateStr)
+		rdate, err := time.Parse(time.RFC3339Nano, retainDateStr)
 		if err != nil {
 			return
 		}
@@ -441,37 +441,44 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		}
 	}
 
-	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
-		return
-	}
-	putOpts := putReplicationOpts(ctx, dest, objInfo)
 	replicationStatus := replication.Complete
+	if rtype != replicateAll {
+		gr.Close()
 
-	// Setup bandwidth throttling
-	peers, _ := globalEndpoints.peers()
-	totalNodesCount := len(peers)
-	if totalNodesCount == 0 {
-		totalNodesCount = 1 // For standalone erasure coding
-	}
-	b := target.BandwidthLimit / int64(totalNodesCount)
-	var headerSize int
-	for k, v := range putOpts.Header() {
-		headerSize += len(k) + len(v)
-	}
-	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
-	if rtype == replicateAll {
-		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
-	} else {
 		// replicate metadata for object tagging/copy with metadata replacement
 		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
 		_, err = tgt.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
-	}
+		if err != nil {
+			replicationStatus = replication.Failed
+		}
+	} else {
+		target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+			gr.Close()
+			return
+		}
 
-	r.Close()
-	if err != nil {
-		replicationStatus = replication.Failed
+		putOpts := putReplicationOpts(ctx, dest, objInfo)
+		// Setup bandwidth throttling
+		peers, _ := globalEndpoints.peers()
+		totalNodesCount := len(peers)
+		if totalNodesCount == 0 {
+			totalNodesCount = 1 // For standalone erasure coding
+		}
+		b := target.BandwidthLimit / int64(totalNodesCount)
+		var headerSize int
+		for k, v := range putOpts.Header() {
+			headerSize += len(k) + len(v)
+		}
+
+		// r takes over closing gr.
+		r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
+		if err != nil {
+			replicationStatus = replication.Failed
+		}
+		r.Close()
 	}
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 	if objInfo.UserTags != "" {

cmd/bucket-targets.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"net/http"
 	"strings"
 	"sync"
@@ -27,6 +28,7 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/madmin"
 	sha256 "github.com/minio/sha256-simd"
@@ -66,7 +68,6 @@ func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType str
 
 // ListBucketTargets - gets list of bucket targets for this bucket.
 func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
-
 	sys.RLock()
 	defer sys.RUnlock()
 
@@ -128,9 +129,13 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	sys.Lock()
 	defer sys.Unlock()
 
-	tgts := sys.targetsMap[bucket]
+	tgts, ok := sys.targetsMap[bucket]
+	if !ok {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+
 	newtgts := make([]madmin.BucketTarget, len(tgts))
-	labels := make(map[string]struct{})
+	labels := make(map[string]struct{}, len(tgts))
 	found := false
 	for idx, t := range tgts {
 		labels[t.Label] = struct{}{}
@@ -196,9 +201,12 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	// delete ARN type from list of matching targets
 	sys.Lock()
 	defer sys.Unlock()
-	targets := make([]madmin.BucketTarget, 0)
 	found := false
-	tgts := sys.targetsMap[bucket]
+	tgts, ok := sys.targetsMap[bucket]
+	if !ok {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	targets := make([]madmin.BucketTarget, 0, len(tgts))
 	for _, tgt := range tgts {
 		if tgt.Arn != arnStr {
 			targets = append(targets, tgt)
@@ -348,7 +356,7 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*m
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
 
 	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
+		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
 	})
 
 	core, err := miniogo.NewCore(tcfg.URL().Host, &miniogo.Options{
@@ -391,3 +399,32 @@ func generateARN(t *madmin.BucketTarget) string {
 	}
 	return arn.String()
 }
+
+// Returns parsed target config. If KMS is configured, remote target is decrypted
+func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) {
+	var (
+		data []byte
+		err  error
+		t    madmin.BucketTargets
+		meta map[string]string
+	)
+	if len(cdata) == 0 {
+		return nil, nil
+	}
+	data = cdata
+	if len(cmetadata) != 0 {
+		if err := json.Unmarshal(cmetadata, &meta); err != nil {
+			return nil, err
+		}
+		if crypto.S3.IsEncrypted(meta) {
+			if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{bucket: bucket, bucketTargetsFile: bucketTargetsFile}); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if err = json.Unmarshal(data, &t); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}

cmd/bucket-versioning.go

@@ -51,6 +51,11 @@ func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, erro
 	return globalBucketMetadataSys.GetVersioningConfig(bucket)
 }
 
+// Reset BucketVersioningSys to initial state.
+func (sys *BucketVersioningSys) Reset() {
+	// There is currently no internal state.
+}
+
 // NewBucketVersioningSys - creates new versioning system.
 func NewBucketVersioningSys() *BucketVersioningSys {
 	return &BucketVersioningSys{}

cmd/common-main.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/color"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -38,15 +39,26 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
 )
 
+// serverDebugLog will enable debug printing
+var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+
 func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
 	rand.Seed(time.Now().UTC().UnixNano())
-	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second)
+
+	initGlobalContext()
+
+	globalReplicationState = newReplicationState()
+	globalTransitionState = newTransitionState()
+
+	console.SetColor("Debug", color.New())
 
 	gob.Register(StorageErr(""))
 }
@@ -64,10 +76,12 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 		}
 	}
 
+	globalCompressConfigMu.Lock()
 	if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
 		logger.Fatal(errInvalidArgument,
 			"Compression support is requested but '%s' does not support compression", name)
 	}
+	globalCompressConfigMu.Unlock()
 }
 
 // Check for updates and print a notification message
@@ -272,6 +286,16 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}
 
+	if env.IsSet(config.EnvRootUser) || env.IsSet(config.EnvRootPassword) {
+		cred, err := auth.CreateCredentials(env.Get(config.EnvRootUser, ""), env.Get(config.EnvRootPassword, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the shell environment")
+		}
+		globalActiveCred = cred
+		globalConfigEncrypted = true
+	}
+
 	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
 		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
 		if err != nil {
@@ -282,6 +306,17 @@ func handleCommonEnvVars() {
 		os.Unsetenv(config.EnvAccessKeyOld)
 		os.Unsetenv(config.EnvSecretKeyOld)
 	}
+
+	if env.IsSet(config.EnvRootUserOld) && env.IsSet(config.EnvRootPasswordOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvRootUserOld, ""), env.Get(config.EnvRootPasswordOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvRootUserOld)
+		os.Unsetenv(config.EnvRootPasswordOld)
+	}
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"sync"
@@ -25,6 +26,7 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/heal"
@@ -57,6 +59,7 @@ func initHelp() {
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
 		config.HealSubSys:           heal.DefaultKVS,
+		config.CrawlerSubSys:        crawler.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -112,6 +115,10 @@ func initHelp() {
 			Key:         config.HealSubSys,
 			Description: "manage object healing frequency and bitrot verification checks",
 		},
+		config.HelpKV{
+			Key:         config.CrawlerSubSys,
+			Description: "manage crawling for usage calculation, lifecycle, healing and more",
+		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
 			Description:     "send server logs to webhook endpoints",
@@ -192,6 +199,7 @@ func initHelp() {
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
 		config.HealSubSys:           heal.Help,
+		config.CrawlerSubSys:        crawler.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -221,6 +229,9 @@ var (
 )
 
 func validateConfig(s config.Config, setDriveCount int) error {
+	// We must have a global lock for this so nobody else modifies env while we do.
+	defer env.LockSetEnv()()
+
 	// Disable merging env values with config for validation.
 	env.SetEnvOff()
 
@@ -249,14 +260,25 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
-	if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
+	compCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+	if err != nil {
 		return err
 	}
+	objAPI := newObjectLayerFn()
+	if objAPI != nil {
+		if compCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
+	}
 
 	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
+	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+		return err
+	}
+
 	{
 		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 		if err != nil {
@@ -438,10 +460,6 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
-	globalHealConfig, err = heal.LookupConfig(s[config.HealSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
-	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
@@ -459,11 +477,6 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
 	}
 
-	globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup Compression: %w", err))
-	}
-
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
@@ -538,6 +551,68 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
+
+	// Apply dynamic config values
+	logger.LogIf(ctx, applyDynamicConfig(ctx, s))
+}
+
+// applyDynamicConfig will apply dynamic config values.
+// Dynamic systems should be in config.SubSystemsDynamic as well.
+func applyDynamicConfig(ctx context.Context, s config.Config) error {
+	// Read all dynamic configs.
+	// API
+	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+	}
+
+	// Compression
+	cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+	if err != nil {
+		return fmt.Errorf("Unable to setup Compression: %w", err)
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI != nil {
+		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
+	}
+
+	// Heal
+	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to apply heal config: %w", err))
+	}
+
+	// Crawler
+	crawlerCfg, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	if err != nil {
+		return fmt.Errorf("Unable to apply crawler config: %w", err)
+	}
+
+	// Apply configurations.
+	// We should not fail after this.
+	globalAPIConfig.init(apiConfig, globalAPIConfig.setDriveCount)
+
+	globalCompressConfigMu.Lock()
+	globalCompressConfig = cmpCfg
+	globalCompressConfigMu.Unlock()
+
+	globalHealConfigMu.Lock()
+	globalHealConfig = healCfg
+	globalHealConfigMu.Unlock()
+
+	logger.LogIf(ctx, crawlerSleeper.Update(crawlerCfg.Delay, crawlerCfg.MaxWait))
+
+	// Update all dynamic config values in memory.
+	globalServerConfigMu.Lock()
+	defer globalServerConfigMu.Unlock()
+	if globalServerConfig != nil {
+		for k := range config.SubSystemsDynamic {
+			globalServerConfig[k] = s[k]
+		}
+	}
+	return nil
 }
 
 // Help - return sub-system level help

cmd/config-encrypted.go

@@ -27,7 +27,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/madmin"
-	etcd "go.etcd.io/etcd/v3/clientv3"
+	etcd "go.etcd.io/etcd/clientv3"
 )
 
 func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
@@ -207,7 +207,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 	}
 
 	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
-		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+		logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
 	}
 
 	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
@@ -294,7 +294,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 	}
 
 	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
-		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+		logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
 	}
 
 	return saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)

cmd/config/api/api.go

@@ -113,8 +113,6 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // acceptable quorum expected for list operations
 func (sCfg Config) GetListQuorum() int {
 	switch sCfg.ListQuorum {
-	case "optimal":
-		return 3
 	case "reduced":
 		return 2
 	case "disk":
@@ -123,7 +121,7 @@ func (sCfg Config) GetListQuorum() int {
 	case "strict":
 		return -1
 	}
-	// Defaults to 3 drives per set.
+	// Defaults to 3 drives per set, defaults to "optimal" value
 	return 3
 }
 

cmd/config/compress/compress.go

@@ -26,19 +26,22 @@ import (
 
 // Config represents the compression settings.
 type Config struct {
-	Enabled    bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
-	MimeTypes  []string `json:"mime-types"`
+	Enabled        bool     `json:"enabled"`
+	AllowEncrypted bool     `json:"allow_encryption"`
+	Extensions     []string `json:"extensions"`
+	MimeTypes      []string `json:"mime-types"`
 }
 
 // Compression environment variables
 const (
-	Extensions = "extensions"
-	MimeTypes  = "mime_types"
+	Extensions     = "extensions"
+	AllowEncrypted = "allow_encryption"
+	MimeTypes      = "mime_types"
 
-	EnvCompressState      = "MINIO_COMPRESS_ENABLE"
-	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
-	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIME_TYPES"
+	EnvCompressState           = "MINIO_COMPRESS_ENABLE"
+	EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
+	EnvCompressExtensions      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes       = "MINIO_COMPRESS_MIME_TYPES"
 
 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
@@ -52,6 +55,10 @@ var (
 			Key:   config.Enable,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   AllowEncrypted,
+			Value: config.EnableOff,
+		},
 		config.KV{
 			Key:   Extensions,
 			Value: DefaultExtensions,
@@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		return cfg, nil
 	}
 
+	allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
+	cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
+	if err != nil {
+		return cfg, err
+	}
+
 	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
 	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
 	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))

cmd/config/config.go

@@ -77,6 +77,7 @@ const (
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
 	HealSubSys           = "heal"
+	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -98,7 +99,7 @@ const (
 )
 
 // SubSystems - all supported sub-systems
-var SubSystems = set.CreateStringSet([]string{
+var SubSystems = set.CreateStringSet(
 	CredentialsSubSys,
 	RegionSubSys,
 	EtcdSubSys,
@@ -113,6 +114,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	CrawlerSubSys,
 	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
@@ -124,7 +126,15 @@ var SubSystems = set.CreateStringSet([]string{
 	NotifyPostgresSubSys,
 	NotifyRedisSubSys,
 	NotifyWebhookSubSys,
-}...)
+)
+
+// SubSystemsDynamic - all sub-systems that have dynamic config.
+var SubSystemsDynamic = set.CreateStringSet(
+	APISubSys,
+	CompressionSubSys,
+	CrawlerSubSys,
+	HealSubSys,
+)
 
 // SubSystemsSingleTargets - subsystems which only support single target.
 var SubSystemsSingleTargets = set.CreateStringSet([]string{
@@ -141,6 +151,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
 	HealSubSys,
+	CrawlerSubSys,
 }...)
 
 // Constant separators
@@ -309,25 +320,29 @@ func (c Config) DelFrom(r io.Reader) error {
 	return nil
 }
 
-// ReadFrom - implements io.ReaderFrom interface
-func (c Config) ReadFrom(r io.Reader) (int64, error) {
+// ReadConfig - read content from input and write into c.
+// Returns whether all parameters were dynamic.
+func (c Config) ReadConfig(r io.Reader) (dynOnly bool, err error) {
 	var n int
 	scanner := bufio.NewScanner(r)
+	dynOnly = true
 	for scanner.Scan() {
 		// Skip any empty lines, or comment like characters
 		text := scanner.Text()
 		if text == "" || strings.HasPrefix(text, KvComment) {
 			continue
 		}
-		if err := c.SetKVS(text, DefaultKVS); err != nil {
-			return 0, err
+		dynamic, err := c.SetKVS(text, DefaultKVS)
+		if err != nil {
+			return false, err
 		}
+		dynOnly = dynOnly && dynamic
 		n += len(text)
 	}
 	if err := scanner.Err(); err != nil {
-		return 0, err
+		return false, err
 	}
-	return int64(n), nil
+	return dynOnly, nil
 }
 
 type configWriteTo struct {
@@ -618,26 +633,27 @@ func (c Config) Clone() Config {
 }
 
 // SetKVS - set specific key values per sub-system.
-func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
+func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) {
 	if len(s) == 0 {
-		return Errorf("input arguments cannot be empty")
+		return false, Errorf("input arguments cannot be empty")
 	}
 	inputs := strings.SplitN(s, KvSpaceSeparator, 2)
 	if len(inputs) <= 1 {
-		return Errorf("invalid number of arguments '%s'", s)
+		return false, Errorf("invalid number of arguments '%s'", s)
 	}
 	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
 	if len(subSystemValue) == 0 {
-		return Errorf("invalid number of arguments %s", s)
+		return false, Errorf("invalid number of arguments %s", s)
 	}
 
 	if !SubSystems.Contains(subSystemValue[0]) {
-		return Errorf("unknown sub-system %s", s)
+		return false, Errorf("unknown sub-system %s", s)
 	}
 
 	if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 {
-		return Errorf("sub-system '%s' only supports single target", subSystemValue[0])
+		return false, Errorf("sub-system '%s' only supports single target", subSystemValue[0])
 	}
+	dynamic = SubSystemsDynamic.Contains(subSystemValue[0])
 
 	tgt := Default
 	subSys := subSystemValue[0]
@@ -647,7 +663,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 
 	fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
 	if len(fields) == 0 {
-		return Errorf("sub-system '%s' cannot have empty keys", subSys)
+		return false, Errorf("sub-system '%s' cannot have empty keys", subSys)
 	}
 
 	var kvs = KVS{}
@@ -670,7 +686,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 			kvs.Set(prevK, madmin.SanitizeValue(kv[1]))
 			continue
 		}
-		return Errorf("key '%s', cannot have empty value", kv[0])
+		return false, Errorf("key '%s', cannot have empty value", kv[0])
 	}
 
 	_, ok := kvs.Lookup(Enable)
@@ -720,11 +736,11 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 			// Return error only if the
 			// key is enabled, for state=off
 			// let it be empty.
-			return Errorf(
+			return false, Errorf(
 				"'%s' is not optional for '%s' sub-system, please check '%s' documentation",
 				hkv.Key, subSys, subSys)
 		}
 	}
 	c[subSys][tgt] = currKVS
-	return nil
+	return dynamic, nil
 }

cmd/config/constants.go

@@ -23,17 +23,21 @@ const (
 
 // Top level common ENVs
 const (
-	EnvAccessKey    = "MINIO_ACCESS_KEY"
-	EnvSecretKey    = "MINIO_SECRET_KEY"
-	EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
-	EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
-	EnvBrowser      = "MINIO_BROWSER"
-	EnvDomain       = "MINIO_DOMAIN"
-	EnvRegionName   = "MINIO_REGION_NAME"
-	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
-	EnvFSOSync      = "MINIO_FS_OSYNC"
-	EnvArgs         = "MINIO_ARGS"
-	EnvDNSWebhook   = "MINIO_DNS_WEBHOOK_ENDPOINT"
+	EnvAccessKey       = "MINIO_ACCESS_KEY"
+	EnvSecretKey       = "MINIO_SECRET_KEY"
+	EnvRootUser        = "MINIO_ROOT_USER"
+	EnvRootPassword    = "MINIO_ROOT_PASSWORD"
+	EnvAccessKeyOld    = "MINIO_ACCESS_KEY_OLD"
+	EnvSecretKeyOld    = "MINIO_SECRET_KEY_OLD"
+	EnvRootUserOld     = "MINIO_ROOT_USER_OLD"
+	EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD"
+	EnvBrowser         = "MINIO_BROWSER"
+	EnvDomain          = "MINIO_DOMAIN"
+	EnvRegionName      = "MINIO_REGION_NAME"
+	EnvPublicIPs       = "MINIO_PUBLIC_IPS"
+	EnvFSOSync         = "MINIO_FS_OSYNC"
+	EnvArgs            = "MINIO_ARGS"
+	EnvDNSWebhook      = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
 	EnvUpdate = "MINIO_UPDATE"
 

cmd/config/crawler/crawler.go

@@ -0,0 +1,88 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawler
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Compression environment variables
+const (
+	Delay   = "delay"
+	MaxWait = "max_wait"
+
+	EnvDelay   = "MINIO_CRAWLER_DELAY"
+	EnvMaxWait = "MINIO_CRAWLER_MAX_WAIT"
+)
+
+// Config represents the heal settings.
+type Config struct {
+	// Delay is the sleep multiplier.
+	Delay float64 `json:"delay"`
+	// MaxWait is maximum wait time between operations
+	MaxWait time.Duration
+}
+
+var (
+	// DefaultKVS - default KV config for heal settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   Delay,
+			Value: "10",
+		},
+		config.KV{
+			Key:   MaxWait,
+			Value: "15s",
+		},
+	}
+
+	// Help provides help for config values
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Delay,
+			Description: `crawler delay multiplier, defaults to '10.0'`,
+			Optional:    true,
+			Type:        "float",
+		},
+		config.HelpKV{
+			Key:         MaxWait,
+			Description: `maximum wait time between operations, defaults to '15s'`,
+			Optional:    true,
+			Type:        "duration",
+		},
+	}
+)
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+	cfg.Delay, err = strconv.ParseFloat(env.Get(EnvDelay, kvs.Get(Delay)), 64)
+	if err != nil {
+		return cfg, err
+	}
+	cfg.MaxWait, err = time.ParseDuration(env.Get(EnvMaxWait, kvs.Get(MaxWait)))
+	if err != nil {
+		return cfg, err
+	}
+	return cfg, nil
+}

cmd/config/dns/etcd_dns.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/coredns/coredns/plugin/etcd/msg"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"go.etcd.io/etcd/v3/clientv3"
+	"go.etcd.io/etcd/clientv3"
 )
 
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)

cmd/config/errors.go

@@ -116,19 +116,19 @@ var (
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",
-		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
+		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
 	)
 
 	ErrInvalidCredentialsBackendEncrypted = newErrFn(
 		"Invalid credentials",
 		"Please set correct credentials in the environment for decryption",
-		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
 	)
 
 	ErrMissingCredentialsBackendEncrypted = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
 	)
 
 	ErrInvalidCredentials = newErrFn(
@@ -140,13 +140,13 @@ var (
 	ErrEnvCredentialsMissingGateway = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
 	)
 
 	ErrEnvCredentialsMissingDistributed = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
 	)
 
 	ErrInvalidErasureEndpoints = newErrFn(

cmd/config/etcd/etcd.go

@@ -25,8 +25,8 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
-	"go.etcd.io/etcd/v3/clientv3"
-	"go.etcd.io/etcd/v3/clientv3/namespace"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3/namespace"
 )
 
 const (

cmd/consolelogger.go

@@ -70,7 +70,7 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerPools EndpointServerP
 // HasLogListeners returns true if console log listeners are registered
 // for this node or peers
 func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
-	return sys != nil && sys.pubsub.HasSubscribers()
+	return sys != nil && sys.pubsub.NumSubscribers() > 0
 }
 
 // Subscribe starts console logging for this node.

cmd/crypto/header.go

@@ -19,231 +19,62 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"net/http"
-	"strings"
 
-	jsoniter "github.com/json-iterator/go"
 	xhttp "github.com/minio/minio/cmd/http"
 )
 
-// SSEHeader is the general AWS SSE HTTP header key.
-const SSEHeader = "X-Amz-Server-Side-Encryption"
-
-const (
-	// SSEKmsID is the HTTP header key referencing the SSE-KMS
-	// key ID.
-	SSEKmsID = SSEHeader + "-Aws-Kms-Key-Id"
-
-	// SSEKmsContext is the HTTP header key referencing the
-	// SSE-KMS encryption context.
-	SSEKmsContext = SSEHeader + "-Context"
-)
-
-const (
-	// SSECAlgorithm is the HTTP header key referencing
-	// the SSE-C algorithm.
-	SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
-
-	// SSECKey is the HTTP header key referencing the
-	// SSE-C client-provided key..
-	SSECKey = SSEHeader + "-Customer-Key"
-
-	// SSECKeyMD5 is the HTTP header key referencing
-	// the MD5 sum of the client-provided key.
-	SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
-)
-
-const (
-	// SSECopyAlgorithm is the HTTP header key referencing
-	// the SSE-C algorithm for SSE-C copy requests.
-	SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
-
-	// SSECopyKey is the HTTP header key referencing the SSE-C
-	// client-provided key for SSE-C copy requests.
-	SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
-
-	// SSECopyKeyMD5 is the HTTP header key referencing the
-	// MD5 sum of the client key for SSE-C copy requests.
-	SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
-)
-
-const (
-	// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
-	// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
-	// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
-	SSEAlgorithmAES256 = "AES256"
-
-	// SSEAlgorithmKMS is the value of 'X-Amz-Server-Side-Encryption' for SSE-KMS.
-	// See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
-	SSEAlgorithmKMS = "aws:kms"
-)
-
 // RemoveSensitiveHeaders removes confidential encryption
 // information - e.g. the SSE-C key - from the HTTP headers.
 // It has the same semantics as RemoveSensitiveEntires.
 func RemoveSensitiveHeaders(h http.Header) {
-	h.Del(SSECKey)
-	h.Del(SSECopyKey)
+	h.Del(xhttp.AmzServerSideEncryptionCustomerKey)
+	h.Del(xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	h.Del(xhttp.AmzMetaUnencryptedContentLength)
 	h.Del(xhttp.AmzMetaUnencryptedContentMD5)
 }
 
-// IsRequested returns true if the HTTP headers indicates
-// that any form server-side encryption (SSE-C, SSE-S3 or SSE-KMS)
-// is requested.
-func IsRequested(h http.Header) bool {
-	return S3.IsRequested(h) || SSEC.IsRequested(h) || SSECopy.IsRequested(h) || S3KMS.IsRequested(h)
-}
-
-// S3 represents AWS SSE-S3. It provides functionality to handle
-// SSE-S3 requests.
-var S3 = s3{}
-
-type s3 struct{}
-
-// IsRequested returns true if the HTTP headers indicates that
-// the S3 client requests SSE-S3.
-func (s3) IsRequested(h http.Header) bool {
-	_, ok := h[SSEHeader]
-	return ok && strings.ToLower(h.Get(SSEHeader)) != SSEAlgorithmKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
-}
-
-// ParseHTTP parses the SSE-S3 related HTTP headers and checks
-// whether they contain valid values.
-func (s3) ParseHTTP(h http.Header) (err error) {
-	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
-		err = ErrInvalidEncryptionMethod
-	}
-	return
-}
-
-// S3KMS represents AWS SSE-KMS. It provides functionality to
-// handle SSE-KMS requests.
-var S3KMS = s3KMS{}
-
-type s3KMS struct{}
-
-// IsRequested returns true if the HTTP headers indicates that
-// the S3 client requests SSE-KMS.
-func (s3KMS) IsRequested(h http.Header) bool {
-	if _, ok := h[SSEKmsID]; ok {
-		return true
-	}
-	if _, ok := h[SSEKmsContext]; ok {
-		return true
-	}
-	if _, ok := h[SSEHeader]; ok {
-		return strings.ToUpper(h.Get(SSEHeader)) != SSEAlgorithmAES256 // Return only true if the SSE header is specified and does not contain the SSE-S3 value
-	}
-	return false
-}
-
-// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
-// and context, if present, on success.
-func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
-	algorithm := h.Get(SSEHeader)
-	if algorithm != SSEAlgorithmKMS {
-		return "", nil, ErrInvalidEncryptionMethod
-	}
-
-	contextStr, ok := h[SSEKmsContext]
-	if ok {
-		var context map[string]interface{}
-		var json = jsoniter.ConfigCompatibleWithStandardLibrary
-		if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
-			return "", nil, err
-		}
-		return h.Get(SSEKmsID), context, nil
-	}
-	return h.Get(SSEKmsID), nil, nil
-}
-
 var (
-	// SSEC represents AWS SSE-C. It provides functionality to handle
-	// SSE-C requests.
-	SSEC = ssec{}
-
 	// SSECopy represents AWS SSE-C for copy requests. It provides
 	// functionality to handle SSE-C copy requests.
 	SSECopy = ssecCopy{}
 )
 
-type ssec struct{}
 type ssecCopy struct{}
 
-// IsRequested returns true if the HTTP headers contains
-// at least one SSE-C header. SSE-C copy headers are ignored.
-func (ssec) IsRequested(h http.Header) bool {
-	if _, ok := h[SSECAlgorithm]; ok {
-		return true
-	}
-	if _, ok := h[SSECKey]; ok {
-		return true
-	}
-	if _, ok := h[SSECKeyMD5]; ok {
-		return true
-	}
-	return false
-}
-
 // IsRequested returns true if the HTTP headers contains
 // at least one SSE-C copy header. Regular SSE-C headers
 // are ignored.
 func (ssecCopy) IsRequested(h http.Header) bool {
-	if _, ok := h[SSECopyAlgorithm]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm]; ok {
 		return true
 	}
-	if _, ok := h[SSECopyKey]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKey]; ok {
 		return true
 	}
-	if _, ok := h[SSECopyKeyMD5]; ok {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5]; ok {
 		return true
 	}
 	return false
 }
 
-// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
-// on success. SSE-C copy headers are ignored.
-func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
-		return key, ErrInvalidCustomerAlgorithm
-	}
-	if h.Get(SSECKey) == "" {
-		return key, ErrMissingCustomerKey
-	}
-	if h.Get(SSECKeyMD5) == "" {
-		return key, ErrMissingCustomerKeyMD5
-	}
-
-	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
-	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
-		return key, ErrInvalidCustomerKey
-	}
-	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
-	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
-		return key, ErrCustomerKeyMD5Mismatch
-	}
-	copy(key[:], clientKey)
-	return key, nil
-}
-
 // ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
 // on success. Regular SSE-C headers are ignored.
 func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != xhttp.AmzEncryptionAES {
 		return key, ErrInvalidCustomerAlgorithm
 	}
-	if h.Get(SSECopyKey) == "" {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey) == "" {
 		return key, ErrMissingCustomerKey
 	}
-	if h.Get(SSECopyKeyMD5) == "" {
+	if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5) == "" {
 		return key, ErrMissingCustomerKeyMD5
 	}
 
-	clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey))
 	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
 		return key, ErrInvalidCustomerKey
 	}
-	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5))
 	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
 		return key, ErrCustomerKeyMD5Mismatch
 	}

cmd/crypto/header_test.go

@@ -18,26 +18,29 @@ import (
 	"net/http"
 	"sort"
 	"testing"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 func TestIsRequested(t *testing.T) {
 	for i, test := range kmsIsRequestedTests {
-		if got := IsRequested(test.Header) && S3KMS.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && S3KMS.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 	for i, test := range s3IsRequestedTests {
-		if got := IsRequested(test.Header) && S3.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && S3.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
 	for i, test := range ssecIsRequestedTests {
-		if got := IsRequested(test.Header) && SSEC.IsRequested(test.Header); got != test.Expected {
-			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
-		}
-	}
-	for i, test := range ssecCopyIsRequestedTests {
-		if got := IsRequested(test.Header) && SSECopy.IsRequested(test.Header); got != test.Expected {
+		_, got := IsRequested(test.Header)
+		got = got && SSEC.IsRequested(test.Header)
+		if got != test.Expected {
 			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
 		}
 	}
@@ -131,11 +134,11 @@ var s3IsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool
 }{
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},         // 0
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},        // 1
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},               // 2
-	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},         // 3
-	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{SSEAlgorithmKMS}}, Expected: false}, // 4
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},                // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true},               // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},                      // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},                // 3
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{xhttp.AmzEncryptionKMS}}, Expected: false}, // 4
 }
 
 func TestS3IsRequested(t *testing.T) {
@@ -403,7 +406,7 @@ func TestSSECopyParse(t *testing.T) {
 		if err == nil && key == zeroKey {
 			t.Errorf("Test %d: parsed client key is zero key", i)
 		}
-		if _, ok := test.Header[SSECKey]; ok {
+		if _, ok := test.Header[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
 			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
 		}
 	}
@@ -414,47 +417,47 @@ var removeSensitiveHeadersTests = []struct {
 }{
 	{
 		Header: http.Header{
-			SSECKey:    []string{""},
-			SSECopyKey: []string{""},
+			xhttp.AmzServerSideEncryptionCustomerKey:     []string{""},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{""},
 		},
 		ExpectedHeader: http.Header{},
 	},
 	{ // Standard SSE-C request headers
 		Header: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 	},
 	{ // Standard SSE-C + SSE-C-copy request headers
 		Header: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm:  []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKey:        []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:     []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm: []string{SSEAlgorithmAES256},
-			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm:  []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:     []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
 		},
 	},
 	{ // Standard SSE-C + metadata request headers
 		Header: http.Header{
-			SSECAlgorithm:       []string{SSEAlgorithmAES256},
-			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			"X-Amz-Meta-Test-1": []string{"Test-1"},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1":                            []string{"Test-1"},
 		},
 		ExpectedHeader: http.Header{
-			SSECAlgorithm:       []string{SSEAlgorithmAES256},
-			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
-			"X-Amz-Meta-Test-1": []string{"Test-1"},
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1":                            []string{"Test-1"},
 		},
 	},
 	{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w

cmd/crypto/metadata.go

@@ -15,19 +15,42 @@
 package crypto
 
 import (
-	"context"
-	"encoding/base64"
-	"errors"
-
 	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
+)
+
+const (
+	// MetaMultipart indicates that the object has been uploaded
+	// in multiple parts - via the S3 multipart API.
+	MetaMultipart = "X-Minio-Internal-Encrypted-Multipart"
+
+	// MetaIV is the random initialization vector (IV) used for
+	// the MinIO-internal key derivation.
+	MetaIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
+
+	// MetaAlgorithm is the algorithm used to derive internal keys
+	// and encrypt the objects.
+	MetaAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
+
+	// MetaSealedKeySSEC is the sealed object encryption key in case of SSE-C.
+	MetaSealedKeySSEC = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
+	// MetaSealedKeyS3 is the sealed object encryption key in case of SSE-S3
+	MetaSealedKeyS3 = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// MetaSealedKeyKMS is the sealed object encryption key in case of SSE-KMS
+	MetaSealedKeyKMS = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"
+
+	// MetaKeyID is the KMS master key ID used to generate/encrypt the data
+	// encryption key (DEK).
+	MetaKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// MetaDataEncryptionKey is the sealed data encryption key (DEK) received from
+	// the KMS.
+	MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
 )
 
 // IsMultiPart returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption
 // and the S3 multipart API.
 func IsMultiPart(metadata map[string]string) bool {
-	if _, ok := metadata[SSEMultipart]; ok {
+	if _, ok := metadata[MetaMultipart]; ok {
 		return true
 	}
 	return false
@@ -37,8 +60,8 @@ func IsMultiPart(metadata map[string]string) bool {
 // information - e.g. the SSE-C key - from the metadata map.
 // It has the same semantics as RemoveSensitiveHeaders.
 func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
-	delete(metadata, SSECKey)
-	delete(metadata, SSECopyKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
 	delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
 }
@@ -46,31 +69,36 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 // RemoveSSEHeaders removes all crypto-specific SSE
 // header entries from the metadata map.
 func RemoveSSEHeaders(metadata map[string]string) {
-	delete(metadata, SSEHeader)
-	delete(metadata, SSEKmsID)
-	delete(metadata, SSEKmsContext)
-	delete(metadata, SSECKeyMD5)
-	delete(metadata, SSECAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryption)
+	delete(metadata, xhttp.AmzServerSideEncryptionKmsID)
+	delete(metadata, xhttp.AmzServerSideEncryptionKmsContext)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCustomerKeyMD5)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
+	delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5)
 }
 
 // RemoveInternalEntries removes all crypto-specific internal
 // metadata entries from the metadata map.
 func RemoveInternalEntries(metadata map[string]string) {
-	delete(metadata, SSEMultipart)
-	delete(metadata, SSEIV)
-	delete(metadata, SSESealAlgorithm)
-	delete(metadata, SSECSealedKey)
-	delete(metadata, S3SealedKey)
-	delete(metadata, S3KMSKeyID)
-	delete(metadata, S3KMSSealedKey)
+	delete(metadata, MetaMultipart)
+	delete(metadata, MetaAlgorithm)
+	delete(metadata, MetaIV)
+	delete(metadata, MetaSealedKeySSEC)
+	delete(metadata, MetaSealedKeyS3)
+	delete(metadata, MetaSealedKeyKMS)
+	delete(metadata, MetaKeyID)
+	delete(metadata, MetaDataEncryptionKey)
 }
 
 // IsSourceEncrypted returns true if the source is encrypted
 func IsSourceEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSECAlgorithm]; ok {
+	if _, ok := metadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
 		return true
 	}
-	if _, ok := metadata[SSEHeader]; ok {
+	if _, ok := metadata[xhttp.AmzServerSideEncryption]; ok {
 		return true
 	}
 	return false
@@ -82,10 +110,10 @@ func IsSourceEncrypted(metadata map[string]string) bool {
 // IsEncrypted only checks whether the metadata contains at least
 // one entry indicating SSE-C or SSE-S3.
 func IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSEIV]; ok {
+	if _, ok := metadata[MetaIV]; ok {
 		return true
 	}
-	if _, ok := metadata[SSESealAlgorithm]; ok {
+	if _, ok := metadata[MetaAlgorithm]; ok {
 		return true
 	}
 	if IsMultiPart(metadata) {
@@ -97,28 +125,7 @@ func IsEncrypted(metadata map[string]string) bool {
 	if SSEC.IsEncrypted(metadata) {
 		return true
 	}
-	return false
-}
-
-// IsEncrypted returns true if the object metadata indicates
-// that the object was uploaded using SSE-S3.
-func (s3) IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[S3SealedKey]; ok {
-		return true
-	}
-	if _, ok := metadata[S3KMSKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[S3KMSSealedKey]; ok {
-		return true
-	}
-	return false
-}
-
-// IsEncrypted returns true if the object metadata indicates
-// that the object was uploaded using SSE-C.
-func (ssec) IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[SSECSealedKey]; ok {
+	if S3KMS.IsEncrypted(metadata) {
 		return true
 	}
 	return false
@@ -129,159 +136,11 @@ func (ssec) IsEncrypted(metadata map[string]string) bool {
 // metadata is nil.
 func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	if metadata == nil {
-		return map[string]string{SSEMultipart: ""}
+		return map[string]string{MetaMultipart: ""}
 	}
-	metadata[SSEMultipart] = ""
+	metadata[MetaMultipart] = ""
 	return metadata
 }
 
-// CreateMetadata encodes the sealed object key into the metadata and returns
-// the modified metadata. If the keyID and the kmsKey is not empty it encodes
-// both into the metadata as well. It allocates a new metadata map if metadata
-// is nil.
-func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
-	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
-	}
-
-	// There are two possibilites:
-	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
-	// - We use a K/V -> There must be no key ID and no KMS data key.
-	// Otherwise, the caller has passed an invalid argument combination.
-	if keyID == "" && len(kmsKey) != 0 {
-		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
-	}
-	if keyID != "" && len(kmsKey) == 0 {
-		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
-	}
-
-	if metadata == nil {
-		metadata = make(map[string]string, 5)
-	}
-
-	metadata[SSESealAlgorithm] = sealedKey.Algorithm
-	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
-	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
-		metadata[S3KMSKeyID] = keyID
-		metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
-	}
-	return metadata
-}
-
-// ParseMetadata extracts all SSE-S3 related values from the object metadata
-// and checks whether they are well-formed. It returns the sealed object key
-// on success. If the metadata contains both, a KMS master key ID and a sealed
-// KMS data key it returns both. If the metadata does not contain neither a
-// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
-// KMS data key. Otherwise, it returns an error.
-func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
-	// Extract all required values from object metadata
-	b64IV, ok := metadata[SSEIV]
-	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalIV
-	}
-	algorithm, ok := metadata[SSESealAlgorithm]
-	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
-	}
-	b64SealedKey, ok := metadata[S3SealedKey]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
-	}
-
-	// There are two possibilites:
-	// - We use a KMS -> There must be a key ID and a KMS data key.
-	// - We use a K/V -> There must be no key ID and no KMS data key.
-	// Otherwise, the metadata is corrupted.
-	keyID, idPresent := metadata[S3KMSKeyID]
-	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
-	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
-	}
-	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
-	}
-
-	// Check whether all extracted values are well-formed
-	var iv [32]byte
-	n, err := base64.StdEncoding.Decode(iv[:], []byte(b64IV))
-	if err != nil || n != 32 {
-		return keyID, kmsKey, sealedKey, errInvalidInternalIV
-	}
-	if algorithm != SealAlgorithm {
-		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
-	}
-	var encryptedKey [64]byte
-	n, err = base64.StdEncoding.Decode(encryptedKey[:], []byte(b64SealedKey))
-	if err != nil || n != 64 {
-		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
-	}
-	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
-		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
-		if err != nil {
-			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
-		}
-	}
-
-	sealedKey.Algorithm = algorithm
-	sealedKey.IV = iv
-	sealedKey.Key = encryptedKey
-	return keyID, kmsKey, sealedKey, nil
-}
-
-// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
-// It allocates a new metadata map if metadata is nil.
-func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
-	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
-	}
-
-	if metadata == nil {
-		metadata = make(map[string]string, 3)
-	}
-	metadata[SSESealAlgorithm] = SealAlgorithm
-	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
-	metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	return metadata
-}
-
-// ParseMetadata extracts all SSE-C related values from the object metadata
-// and checks whether they are well-formed. It returns the sealed object key
-// on success.
-func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
-	// Extract all required values from object metadata
-	b64IV, ok := metadata[SSEIV]
-	if !ok {
-		return sealedKey, errMissingInternalIV
-	}
-	algorithm, ok := metadata[SSESealAlgorithm]
-	if !ok {
-		return sealedKey, errMissingInternalSealAlgorithm
-	}
-	b64SealedKey, ok := metadata[SSECSealedKey]
-	if !ok {
-		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
-	}
-
-	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
-		return sealedKey, errInvalidInternalIV
-	}
-	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
-		return sealedKey, errInvalidInternalSealAlgorithm
-	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
-	}
-
-	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
-	return sealedKey, nil
-}
-
 // IsETagSealed returns true if the etag seems to be encrypted.
 func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -27,9 +27,9 @@ var isMultipartTests = []struct {
 	Metadata  map[string]string
 	Multipart bool
 }{
-	{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}},                           // 0
+	{Multipart: true, Metadata: map[string]string{MetaMultipart: ""}},                          // 0
 	{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
-	{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}},                 // 2
+	{Multipart: true, Metadata: map[string]string{MetaMultipart: "some-value"}},                // 2
 	{Multipart: false, Metadata: map[string]string{"": ""}},                                    // 3
 	{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
 }
@@ -46,13 +46,13 @@ var isEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}},                               // 0
-	{Encrypted: true, Metadata: map[string]string{SSEIV: ""}},                                      // 1
-	{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}},                           // 2
-	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
-	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
-	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
-	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: true, Metadata: map[string]string{MetaMultipart: ""}},                              // 0
+	{Encrypted: true, Metadata: map[string]string{MetaIV: ""}},                                     // 1
+	{Encrypted: true, Metadata: map[string]string{MetaAlgorithm: ""}},                              // 2
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                          // 3
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
+	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
+	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -69,13 +69,13 @@ var s3IsEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
-	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
-	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
-	{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}},                             // 3
-	{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}},                                // 4
-	{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}},                                 // 5
-	{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}},                             // 6
+	{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}},                             // 0
+	{Encrypted: false, Metadata: map[string]string{MetaIV: ""}},                                    // 1
+	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
+	{Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                         // 3
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
+	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
+	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -92,13 +92,13 @@ var ssecIsEncryptedTests = []struct {
 	Metadata  map[string]string
 	Encrypted bool
 }{
-	{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}},                              // 0
-	{Encrypted: false, Metadata: map[string]string{SSEIV: ""}},                                     // 1
-	{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}},                          // 2
-	{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}},                              // 3
-	{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}},                               // 4
-	{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}},                                // 5
-	{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}},                            // 6
+	{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}},                             // 0
+	{Encrypted: false, Metadata: map[string]string{MetaIV: ""}},                                    // 1
+	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
+	{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                          // 3
+	{Encrypted: false, Metadata: map[string]string{MetaSealedKeyS3: ""}},                           // 4
+	{Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}},                                 // 5
+	{Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                     // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }
@@ -121,65 +121,65 @@ var s3ParseMetadataTests = []struct {
 }{
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
 	{
-		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
+		ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaDataEncryptionKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
 	{
 		ExpectedErr: errInvalidInternalIV,
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 5
 	{
 		ExpectedErr: errInvalidInternalSealAlgorithm,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
 		ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
-			S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeyS3: "",
+			MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
 		ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
-			S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "key-1",
+			MetaDataEncryptionKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
 		},
 		DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
 	}, // 8
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "", MetaDataEncryptionKey: "",
 		},
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
 	}, // 9
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
-			S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
-			S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
+			MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeyS3: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), MetaKeyID: "key-1",
+			MetaDataEncryptionKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
 		},
 		DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
 	}, // 10
@@ -223,43 +223,43 @@ var ssecParseMetadataTests = []struct {
 
 	SealedKey SealedKey
 }{
-	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
-	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
+	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                      // 0
+	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
 		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
 		ExpectedErr: errInvalidInternalIV,
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
+		Metadata:    map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeySSEC: ""}, SealedKey: SealedKey{},
 	}, // 3
 	{
 		ExpectedErr: errInvalidInternalSealAlgorithm,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeySSEC: "",
 		},
 		SealedKey: SealedKey{},
 	}, // 4
 	{
 		ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeySSEC: "",
 		},
 		SealedKey: SealedKey{},
 	}, // 5
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
-			SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
+			MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(make([]byte, 64)),
 		},
 		SealedKey: SealedKey{Algorithm: SealAlgorithm},
 	}, // 6
 	{
 		ExpectedErr: nil,
 		Metadata: map[string]string{
-			SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
-			SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
+			MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: InsecureSealAlgorithm,
+			MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
 		},
 		SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
 	}, // 7
@@ -267,8 +267,8 @@ var ssecParseMetadataTests = []struct {
 
 func TestCreateMultipartMetadata(t *testing.T) {
 	metadata := CreateMultipartMetadata(nil)
-	if v, ok := metadata[SSEMultipart]; !ok || v != "" {
-		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
+	if v, ok := metadata[MetaMultipart]; !ok || v != "" {
+		t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", MetaMultipart, v, "")
 	}
 }
 
@@ -411,20 +411,20 @@ var removeInternalEntriesTests = []struct {
 }{
 	{ // 0
 		Metadata: map[string]string{
-			SSEMultipart:     "",
-			SSEIV:            "",
-			SSESealAlgorithm: "",
-			SSECSealedKey:    "",
-			S3SealedKey:      "",
-			S3KMSKeyID:       "",
-			S3KMSSealedKey:   "",
+			MetaMultipart:         "",
+			MetaIV:                "",
+			MetaAlgorithm:         "",
+			MetaSealedKeySSEC:     "",
+			MetaSealedKeyS3:       "",
+			MetaKeyID:             "",
+			MetaDataEncryptionKey: "",
 		},
 		Expected: map[string]string{},
 	},
 	{ // 1
 		Metadata: map[string]string{
-			SSEMultipart:         "",
-			SSEIV:                "",
+			MetaMultipart:        "",
+			MetaIV:               "",
 			"X-Amz-Meta-A":       "X",
 			"X-Minio-Internal-B": "Y",
 		},

cmd/crypto/sse-c.go

@@ -0,0 +1,157 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"net/http"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type ssec struct{}
+
+var (
+	// SSEC represents AWS SSE-C. It provides functionality to handle
+	// SSE-C requests.
+	SSEC = ssec{}
+
+	_ Type = SSEC
+)
+
+// String returns the SSE domain as string. For SSE-C the
+// domain is "SSE-C".
+func (ssec) String() string { return "SSE-C" }
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-C header. SSE-C copy headers are ignored.
+func (ssec) IsRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKeyMD5]; ok {
+		return true
+	}
+	return false
+}
+
+// IsEncrypted returns true if the metadata contains an SSE-C
+// entry inidicating that the object has been encrypted using
+// SSE-C.
+func (ssec) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeySSEC]; ok {
+		return true
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
+// on success. SSE-C copy headers are ignored.
+func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm) != xhttp.AmzEncryptionAES {
+		return key, ErrInvalidCustomerAlgorithm
+	}
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerKey) == "" {
+		return key, ErrMissingCustomerKey
+	}
+	if h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5) == "" {
+		return key, ErrMissingCustomerKeyMD5
+	}
+
+	clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKey))
+	if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
+		return key, ErrInvalidCustomerKey
+	}
+	keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
+	if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
+		return key, ErrCustomerKeyMD5Mismatch
+	}
+	copy(key[:], clientKey)
+	return key, nil
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (s3 ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := s3.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// CreateMetadata encodes the sealed key into the metadata
+// and returns the modified metadata. It allocates a new
+// metadata map if metadata is nil.
+func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 3)
+	}
+	metadata[MetaAlgorithm] = SealAlgorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeySSEC] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-C related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success.
+func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeySSEC]
+	if !ok {
+		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
+		return sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return sealedKey, nil
+}

cmd/crypto/sse-kms.go

@@ -0,0 +1,201 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"path"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type ssekms struct{}
+
+var (
+	// S3KMS represents AWS SSE-KMS. It provides functionality to
+	// handle SSE-KMS requests.
+	S3KMS = ssekms{}
+
+	_ Type = S3KMS
+)
+
+// String returns the SSE domain as string. For SSE-KMS the
+// domain is "SSE-KMS".
+func (ssekms) String() string { return "SSE-KMS" }
+
+// IsRequested returns true if the HTTP headers contains
+// at least one SSE-KMS header.
+func (ssekms) IsRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
+		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
+	}
+	return false
+}
+
+// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
+// and the KMS context on success.
+func (ssekms) ParseHTTP(h http.Header) (string, Context, error) {
+	algorithm := h.Get(xhttp.AmzServerSideEncryption)
+	if algorithm != xhttp.AmzEncryptionKMS {
+		return "", nil, ErrInvalidEncryptionMethod
+	}
+
+	var ctx Context
+	if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		if err := json.Unmarshal([]byte(context[0]), &ctx); err != nil {
+			return "", nil, err
+		}
+	}
+	return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-KMS.
+func (ssekms) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeyKMS]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return true
+	}
+	return false
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (s3 ssekms) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	if err != nil {
+		return key, err
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return key, err
+	}
+	err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
+	return key, err
+}
+
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
+func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 5)
+	}
+
+	metadata[MetaAlgorithm] = sealedKey.Algorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[MetaKeyID] = keyID
+		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-KMS related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
+func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[MetaKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+	}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+	}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}

cmd/crypto/sse-s3.go

@@ -0,0 +1,181 @@
+/*
+ * Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crypto
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"net/http"
+	"path"
+	"strings"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+)
+
+type sses3 struct{}
+
+var (
+	// S3 represents AWS SSE-S3. It provides functionality to handle
+	// SSE-S3 requests.
+	S3 = sses3{}
+
+	_ Type = S3
+)
+
+// String returns the SSE domain as string. For SSE-S3 the
+// domain is "SSE-S3".
+func (sses3) String() string { return "SSE-S3" }
+
+func (sses3) IsRequested(h http.Header) bool {
+	_, ok := h[xhttp.AmzServerSideEncryption]
+	return ok && strings.ToLower(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
+}
+
+// ParseHTTP parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (sses3) ParseHTTP(h http.Header) error {
+	if h.Get(xhttp.AmzServerSideEncryption) != xhttp.AmzEncryptionAES {
+		return ErrInvalidEncryptionMethod
+	}
+	return nil
+}
+
+// IsEncrypted returns true if the object metadata indicates
+// that the object was uploaded using SSE-S3.
+func (sses3) IsEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[MetaSealedKeyS3]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return true
+	}
+	return false
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (s3 sses3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	if err != nil {
+		return key, err
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return key, err
+	}
+	err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
+	return key, err
+}
+
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
+func (sses3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
+	if sealedKey.Algorithm != SealAlgorithm {
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
+	if metadata == nil {
+		metadata = make(map[string]string, 5)
+	}
+
+	metadata[MetaAlgorithm] = sealedKey.Algorithm
+	metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
+	metadata[MetaSealedKeyS3] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[MetaKeyID] = keyID
+		metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
+	return metadata
+}
+
+// ParseMetadata extracts all SSE-S3 related values from the object metadata
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
+func (sses3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+	// Extract all required values from object metadata
+	b64IV, ok := metadata[MetaIV]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalIV
+	}
+	algorithm, ok := metadata[MetaAlgorithm]
+	if !ok {
+		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+	}
+	b64SealedKey, ok := metadata[MetaSealedKeyS3]
+	if !ok {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+	}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[MetaKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+	}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+	}
+
+	// Check whether all extracted values are well-formed
+	iv, err := base64.StdEncoding.DecodeString(b64IV)
+	if err != nil || len(iv) != 32 {
+		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+	}
+	if algorithm != SealAlgorithm {
+		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+	}
+	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
+	if err != nil || len(encryptedKey) != 64 {
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+	}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
+	}
+
+	sealedKey.Algorithm = algorithm
+	copy(sealedKey.IV[:], iv)
+	copy(sealedKey.Key[:], encryptedKey)
+	return keyID, kmsKey, sealedKey, nil
+}

cmd/crypto/sse.go

@@ -17,44 +17,15 @@ package crypto
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
-	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/sio"
 )
 
-const (
-	// SSEMultipart is the metadata key indicating that the object
-	// was uploaded using the S3 multipart API and stored using
-	// some from of server-side-encryption.
-	SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
-
-	// SSEIV is the metadata key referencing the random initialization
-	// vector (IV) used for SSE-S3 and SSE-C key derivation.
-	SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
-
-	// SSESealAlgorithm is the metadata key referencing the algorithm
-	// used by SSE-C and SSE-S3 to encrypt the object.
-	SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
-
-	// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
-	SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
-
-	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
-	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
-
-	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
-	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
-	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
-
-	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
-	// by KMS. It is only used for SSE-S3 + KMS.
-	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
-)
-
 const (
 	// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
 	// the key-encryption-key and to en/decrypt the object data.
@@ -67,39 +38,34 @@ const (
 	InsecureSealAlgorithm = "DARE-SHA256"
 )
 
-// String returns the SSE domain as string. For SSE-S3 the
-// domain is "SSE-S3".
-func (s3) String() string { return "SSE-S3" }
+// Type represents an AWS SSE type:
+//  • SSE-C
+//  • SSE-S3
+//  • SSE-KMS
+type Type interface {
+	fmt.Stringer
 
-// UnsealObjectKey extracts and decrypts the sealed object key
-// from the metadata using KMS and returns the decrypted object
-// key.
-func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
-	if err != nil {
-		return
-	}
-	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
-	if err != nil {
-		return
-	}
-	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
-	return
+	IsRequested(http.Header) bool
+
+	IsEncrypted(map[string]string) bool
 }
 
-// String returns the SSE domain as string. For SSE-C the
-// domain is "SSE-C".
-func (ssec) String() string { return "SSE-C" }
-
-// UnsealObjectKey extracts and decrypts the sealed object key
-// from the metadata using the SSE-C client key of the HTTP headers
-// and returns the decrypted object key.
-func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	clientKey, err := sse.ParseHTTP(h)
-	if err != nil {
-		return
+// IsRequested returns true and the SSE Type if the HTTP headers
+// indicate that some form server-side encryption is requested.
+//
+// If no SSE headers are present then IsRequested returns false
+// and no Type.
+func IsRequested(h http.Header) (Type, bool) {
+	switch {
+	case S3.IsRequested(h):
+		return S3, true
+	case S3KMS.IsRequested(h):
+		return S3KMS, true
+	case SSEC.IsRequested(h):
+		return SSEC, true
+	default:
+		return nil, false
 	}
-	return unsealObjectKey(clientKey, metadata, bucket, object)
 }
 
 // UnsealObjectKey extracts and decrypts the sealed object key

cmd/data-crawler.go

@@ -21,20 +21,20 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"math"
 	"math/rand"
 	"os"
 	"path"
-	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/heal"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
@@ -43,8 +43,7 @@ import (
 
 const (
 	dataCrawlSleepPerFolder  = time.Millisecond // Time to wait between folders.
-	dataCrawlSleepDefMult    = 10.0             // Default multiplier for waits between operations.
-	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
+	dataCrawlStartDelay      = 1 * time.Minute  // Time to wait on startup and between cycles.
 	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
 
 	healDeleteDangling    = true
@@ -53,15 +52,17 @@ const (
 )
 
 var (
-	globalHealConfig             heal.Config
+	globalHealConfig   heal.Config
+	globalHealConfigMu sync.Mutex
+
 	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	// Sleeper values are updated when config is loaded.
+	crawlerSleeper = newDynamicSleeper(10, 10*time.Second)
 )
 
-// initDataCrawler will start the crawler unless disabled.
+// initDataCrawler will start the crawler in the background.
 func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-		go runDataCrawler(ctx, objAPI)
-	}
+	go runDataCrawler(ctx, objAPI)
 }
 
 // runDataCrawler will start a data crawler.
@@ -95,11 +96,21 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
+	crawlTimer := time.NewTimer(dataCrawlStartDelay)
+	defer crawlTimer.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.NewTimer(dataCrawlStartDelay).C:
+		case <-crawlTimer.C:
+			// Reset the timer for next cycle.
+			crawlTimer.Reset(dataCrawlStartDelay)
+
+			if intDataUpdateTracker.debug {
+				console.Debugln("starting crawler cycle")
+			}
+
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
@@ -141,7 +152,6 @@ type folderScanner struct {
 	newCache   dataUsageCache
 	withFilter *bloomFilter
 
-	dataUsageCrawlMult  float64
 	dataUsageCrawlDebug bool
 	healFolderInclude   uint32 // Include a clean folder one in n cycles.
 	healObjectSelect    uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
@@ -159,10 +169,10 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
 	if intDataUpdateTracker.debug {
 		defer func() {
-			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
+			console.Debugf(logPrefix+" Crawl time: %v %s\n", time.Since(t), logSuffix)
 		}()
 
 	}
@@ -172,11 +182,6 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		return cache, errors.New("internal error: root scan attempted")
 	}
 
-	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		delayMult = dataCrawlSleepDefMult
-	}
 	s := folderScanner{
 		root:                basePath,
 		getSize:             getSize,
@@ -184,7 +189,6 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		newCache:            dataUsageCache{Info: cache.Info},
 		newFolders:          nil,
 		existingFolders:     nil,
-		dataUsageCrawlMult:  delayMult,
 		dataUsageCrawlDebug: intDataUpdateTracker.debug,
 		healFolderInclude:   0,
 		healObjectSelect:    0,
@@ -196,7 +200,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		if ok {
 			s.disks = objAPI.GetDisksID(cache.Disks...)
 			if len(s.disks) != len(cache.Disks) {
-				logger.Info(logPrefix+"Missing disks, want %d, found %d. Cannot heal."+logSuffix, len(cache.Disks), len(s.disks))
+				console.Debugf(logPrefix+"Missing disks, want %d, found %d. Cannot heal. %s\n", len(cache.Disks), len(s.disks), logSuffix)
 				s.disks = s.disks[:0]
 			}
 		}
@@ -211,28 +215,28 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 	if len(cache.Info.BloomFilter) > 0 {
 		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
+		_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
 		if err != nil {
 			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
 			s.withFilter = nil
 		}
 	}
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
+		console.Debugf(logPrefix+"Start crawling. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
 	}
 
 	done := ctx.Done()
 	var flattenLevels = 2
 
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
+		console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
 	}
 
 	// Always scan flattenLevels deep. Cache root is level 0.
 	todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}}
 	for i := 0; i < flattenLevels; i++ {
 		if s.dataUsageCrawlDebug {
-			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
+			console.Debugf(logPrefix+"Level %v, scanning %v directories. %s\n", i, len(todo), logSuffix)
 		}
 		select {
 		case <-done:
@@ -248,7 +252,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+		console.Debugf(logPrefix+"New folders: %v %s\n", s.newFolders, logSuffix)
 	}
 
 	// Add new folders first
@@ -264,7 +268,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 			continue
 		}
 		if du == nil {
-			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
+			console.Debugln(logPrefix + "no disk usage provided" + logSuffix)
 			continue
 		}
 
@@ -277,7 +281,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+		console.Debugf(logPrefix+"Existing folders: %v %s\n", len(s.existingFolders), logSuffix)
 	}
 
 	// Do selective scanning of existing folders.
@@ -304,13 +308,13 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 				if !s.withFilter.containsDir(folder.name) {
 					if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
 						if s.dataUsageCrawlDebug {
-							logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+							console.Debugf(logPrefix+"Skipping non-updated folder: %v %s\n", folder, logSuffix)
 						}
 						s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
 						continue
 					} else {
 						if s.dataUsageCrawlDebug {
-							logger.Info(logPrefix+"Adding non-updated folder to heal check: %v"+logSuffix, folder.name)
+							console.Debugf(logPrefix+"Adding non-updated folder to heal check: %v %s\n", folder.name, logSuffix)
 						}
 						// Update probability of including objects
 						folder.objectHealProbDiv = s.healFolderInclude
@@ -332,7 +336,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		s.newCache.replaceHashed(h, folder.parent, *du)
 	}
 	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
+		console.Debugf(logPrefix+"Finished crawl, %v entries %s\n", len(s.newCache.Cache), logSuffix)
 	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle++
@@ -346,6 +350,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
 	var nextFolders []cachedFolder
 	done := ctx.Done()
+	scannerLogPrefix := color.Green("folder-scanner:")
 	for _, folder := range folders {
 		select {
 		case <-done:
@@ -357,16 +362,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 
 		// If there are lifecycle rules for the prefix, remove the filter.
 		filter := f.withFilter
+		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil {
-			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
-			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				}
-				activeLifeCycle = f.oldCache.Info.lifeCycle
-				filter = nil
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+			if f.dataUsageCrawlDebug {
+				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
+			activeLifeCycle = f.oldCache.Info.lifeCycle
+			filter = nil
 		}
 		if _, ok := f.oldCache.Cache[thisHash.Key()]; filter != nil && ok {
 			// If folder isn't in filter and we have data, skip it completely.
@@ -374,19 +377,19 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("folder-scanner:")+" Skipping non-updated folder: %v", folder.name)
+						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
 					}
 					continue
 				} else {
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("folder-scanner:")+" Adding non-updated folder to heal check: %v", folder.name)
+						console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
 					}
 					// If probability was already crawlerHealFolderInclude, keep it.
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 		}
-		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+		crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 
 		cache := dataUsageEntry{}
 
@@ -396,14 +399,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
 			if bucket == "" {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" no bucket (%s,%s)", f.root, entName)
+					console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
 				}
 				return nil
 			}
 
 			if isReservedOrInvalidBucket(bucket, false) {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" invalid bucket: %v, entry: %v", bucket, entName)
+					console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName)
 				}
 				return nil
 			}
@@ -420,8 +423,11 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				cache.addChildString(entName)
 
 				this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
-				delete(existing, h.Key())
+
+				delete(existing, h.Key()) // h.Key() already accounted for.
+
 				cache.addChild(h)
+
 				if final {
 					if exists {
 						f.existingFolders = append(f.existingFolders, this)
@@ -435,7 +441,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			}
 
 			// Dynamic time delay.
-			t := UTCNow()
+			wait := crawlerSleeper.Timer(ctx)
 
 			// Get file size, ignore errors.
 			item := crawlItem{
@@ -446,18 +452,31 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
-				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
+				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
-			size, err := f.getSize(item)
 
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			sizeSummary, err := f.getSize(item)
 			if err == errSkipFile {
+				wait() // wait to proceed to next entry.
+
 				return nil
 			}
-			logger.LogIf(ctx, err)
-			cache.Size += size
+
+			// successfully read means we have a valid object.
+
+			// Remove filename i.e is the meta file to construct object name
+			item.transformMetaDir()
+
+			// Object already accounted for, remove from heal map,
+			// simply because getSize() function already heals the
+			// object.
+			delete(existing, path.Join(item.bucket, item.objectPath()))
+
+			cache.addSizes(sizeSummary)
 			cache.Objects++
-			cache.ObjSizes.add(size)
+			cache.ObjSizes.add(sizeSummary.totalSize)
+
+			wait() // wait to proceed to next entry.
 
 			return nil
 		})
@@ -499,19 +518,19 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			bucket:    "",
 		}
 
+		healObjectsPrefix := color.Green("healObjects:")
 		for k := range existing {
 			bucket, prefix := path2BucketObject(k)
 			if f.dataUsageCrawlDebug {
-				logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
+				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
 			}
 
 			// Dynamic time delay.
-			t := UTCNow()
-
+			wait := crawlerSleeper.Timer(ctx)
 			resolver.bucket = bucket
 
 			foundObjs := false
-			dangling := true
+			dangling := false
 			ctx, cancel := context.WithCancel(ctx)
 			err := listPathRaw(ctx, listPathRawOptions{
 				disks:          f.disks,
@@ -523,26 +542,25 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				// Weird, maybe transient error.
 				agreed: func(entry metaCacheEntry) {
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("healObjects:")+" got agreement: %v", entry.name)
-					}
-					if entry.isObject() {
-						dangling = false
+						console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
 					}
 				},
 				// Some disks have data for this.
 				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("healObjects:")+" got partial, %d agreed, errs: %v", nAgreed, errs)
+						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
 					}
+
+					// agreed value less than expected quorum
+					dangling = nAgreed < resolver.objQuorum || nAgreed < resolver.dirQuorum
+
 					// Sleep and reset.
-					sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-					t = UTCNow()
+					wait()
+					wait = crawlerSleeper.Timer(ctx)
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						for _, err := range errs {
 							if err != nil {
-								// Not all disks are ready, do nothing for now.
-								dangling = false
 								return
 							}
 						}
@@ -552,12 +570,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 					}
 
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("healObjects:")+" resolved to: %v, dir: %v", entry.name, entry.isDir())
+						console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
 					}
+
 					if entry.isDir() {
 						return
 					}
-					dangling = false
 					// We got an entry which we should be able to heal.
 					fiv, err := entry.fileInfoVersions(bucket)
 					if err != nil {
@@ -566,45 +584,54 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 							object:    entry.name,
 							versionID: "",
 						}, madmin.HealItemObject)
-						logger.LogIf(ctx, err)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
 						foundObjs = foundObjs || err == nil
 						return
 					}
 					for _, ver := range fiv.Versions {
 						// Sleep and reset.
-						sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-						t = UTCNow()
+						wait()
+						wait = crawlerSleeper.Timer(ctx)
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    fiv.Name,
 							versionID: ver.VersionID,
 						}, madmin.HealItemObject)
-						logger.LogIf(ctx, err)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
 						foundObjs = foundObjs || err == nil
 					}
 				},
 				// Too many disks failed.
 				finished: func(errs []error) {
 					if f.dataUsageCrawlDebug {
-						logger.Info(color.Green("healObjects:")+" too many errors: %v", errs)
+						console.Debugf(healObjectsPrefix+" too many errors: %v\n", errs)
 					}
-					dangling = false
 					cancel()
 				},
 			})
 
 			if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound {
-				logger.Info(color.Green("healObjects:")+" checking returned value %v (%T)", err, err)
+				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
 
 			// If we found one or more disks with this folder, delete it.
 			if err == nil && dangling {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("healObjects:")+" deleting dangling directory %s", prefix)
+					console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
 				}
-				// If we have quorum, found directories, but no objects, issue heal to delete the dangling.
-				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: true},
+
+				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
+					Recursive: true,
+					Remove:    true,
+				},
 					func(bucket, object, versionID string) error {
+						// Wait for each heal as per crawler frequency.
+						wait()
+						wait = crawlerSleeper.Timer(ctx)
 						return bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    object,
@@ -613,7 +640,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 					})
 			}
 
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			wait()
 
 			// Add unless healing returned an error.
 			if foundObjs {
@@ -640,6 +667,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 	var addDir func(entName string, typ os.FileMode) error
 	var dirStack = []string{f.root, folder.name}
 
+	deepScannerLogPrefix := color.Green("deep-scanner:")
 	addDir = func(entName string, typ os.FileMode) error {
 		select {
 		case <-done:
@@ -651,12 +679,12 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
 			dirStack = dirStack[:len(dirStack)-1]
-			sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+			crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 			return err
 		}
 
 		// Dynamic time delay.
-		t := UTCNow()
+		wait := crawlerSleeper.Timer(ctx)
 
 		// Get file size, ignore errors.
 		dirStack = append(dirStack, entName)
@@ -665,16 +693,14 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 
 		bucket, prefix := path2BucketObjectWithBasePath(f.root, fileName)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil {
-			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
-				}
-				activeLifeCycle = f.oldCache.Info.lifeCycle
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
+			if f.dataUsageCrawlDebug {
+				console.Debugf(deepScannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
+			activeLifeCycle = f.oldCache.Info.lifeCycle
 		}
 
-		size, err := f.getSize(
+		sizeSummary, err := f.getSize(
 			crawlItem{
 				Path:       fileName,
 				Typ:        typ,
@@ -683,19 +709,24 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
-				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
+				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			})
 
-		// Don't sleep for really small amount of time
-		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-
 		if err == errSkipFile {
+			// Wait to throttle IO
+			wait()
+
 			return nil
 		}
+
 		logger.LogIf(ctx, err)
-		cache.Size += size
+		cache.addSizes(sizeSummary)
 		cache.Objects++
-		cache.ObjSizes.add(size)
+		cache.ObjSizes.add(sizeSummary.totalSize)
+
+		// Wait to throttle IO
+		wait()
+
 		return nil
 	}
 	err := readDirFn(path.Join(dirStack...), addDir)
@@ -718,7 +749,15 @@ type crawlItem struct {
 	debug      bool
 }
 
-type getSizeFn func(item crawlItem) (int64, error)
+type sizeSummary struct {
+	totalSize      int64
+	replicatedSize int64
+	pendingSize    int64
+	failedSize     int64
+	replicaSize    int64
+}
+
+type getSizeFn func(item crawlItem) (sizeSummary, error)
 
 // transformMetaDir will transform a directory to prefix/file.ext
 func (i *crawlItem) transformMetaDir() {
@@ -737,6 +776,7 @@ type actionMeta struct {
 	oi               ObjectInfo
 	successorModTime time.Time // The modtime of the successor version
 	numVersions      int       // The number of versions of this object
+	bitRotScan       bool      // indicates if bitrot check was requested.
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -748,11 +788,20 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
+	applyActionsLogPrefix := color.Green("applyActions:")
 	if i.heal {
 		if i.debug {
-			logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID)
+			if meta.oi.VersionID != "" {
+				console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), meta.oi.VersionID)
+			} else {
+				console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
+			}
 		}
-		res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling})
+		healOpts := madmin.HealOpts{Remove: healDeleteDangling}
+		if meta.bitRotScan {
+			healOpts.ScanMode = madmin.HealDeepScan
+		}
+		res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, healOpts)
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return 0
 		}
@@ -763,6 +812,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		size = res.ObjectSize
 	}
 	if i.lifeCycle == nil {
+		if i.debug {
+			console.Debugf(applyActionsLogPrefix+" no lifecycle rules to apply: %q\n", i.objectPath())
+		}
 		return size
 	}
 
@@ -782,7 +834,11 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			TransitionStatus: meta.oi.TransitionStatus,
 		})
 	if i.debug {
-		logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
+		if versionID != "" {
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
+		} else {
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
+		}
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
@@ -790,6 +846,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 	default:
 		// No action.
+		if i.debug {
+			console.Debugf(applyActionsLogPrefix+" object not expirable: %q\n", i.objectPath())
+		}
 		return size
 	}
 
@@ -830,9 +889,8 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		TransitionStatus: obj.TransitionStatus,
 	}
 	action = i.lifeCycle.ComputeAction(lcOpts)
-
 	if i.debug {
-		logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
+		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
@@ -854,7 +912,11 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			locked := enforceRetentionForDeletion(ctx, obj)
 			if locked {
 				if i.debug {
-					logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath())
+					if obj.VersionID != "" {
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", i.objectPath(), obj.VersionID)
+					} else {
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", i.objectPath())
+					}
 				}
 				return size
 			}
@@ -876,18 +938,21 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		globalTransitionState.queueTransitionTask(obj)
 		return 0
 	}
+
 	if obj.TransitionStatus != "" {
 		if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), lcOpts, action, false); err != nil {
 			logger.LogIf(ctx, err)
 			return size
 		}
-	} else {
-		obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
-		if err != nil {
-			// Assume it is still there.
-			logger.LogIf(ctx, err)
-			return size
-		}
+		// Notification already sent at *deleteTransitionedObject*, return '0' here.
+		return 0
+	}
+
+	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
+	if err != nil {
+		// Assume it is still there.
+		logger.LogIf(ctx, err)
+		return size
 	}
 
 	eventName := event.ObjectRemovedDelete
@@ -910,60 +975,176 @@ func (i *crawlItem) objectPath() string {
 	return path.Join(i.prefix, i.objectName)
 }
 
-// sleepDuration multiplies the duration d by x
-// and sleeps if is more than 100 micro seconds.
-// Sleep is limited to max 15 seconds.
-func sleepDuration(d time.Duration, x float64) {
-	const maxWait = 15 * time.Second
-	const minWait = 100 * time.Microsecond
-	// Don't sleep for really small amount of time
-	if d := time.Duration(float64(d) * x); d > minWait {
-		if d > maxWait {
-			d = maxWait
-		}
-		time.Sleep(d)
-	}
-}
-
 // healReplication will heal a scanned item that has failed replication.
-func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta) {
-	if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
-		//heal delete marker replication failure or versioned delete replication failure
-		if meta.oi.ReplicationStatus == replication.Pending ||
-			meta.oi.ReplicationStatus == replication.Failed ||
-			meta.oi.VersionPurgeStatus == Failed || meta.oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, meta)
+func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+		// heal delete marker replication failure or versioned delete replication failure
+		if oi.ReplicationStatus == replication.Pending ||
+			oi.ReplicationStatus == replication.Failed ||
+			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
+			i.healReplicationDeletes(ctx, o, oi)
 			return
 		}
 	}
-	if meta.oi.ReplicationStatus == replication.Pending ||
-		meta.oi.ReplicationStatus == replication.Failed {
-		globalReplicationState.queueReplicaTask(meta.oi)
+	switch oi.ReplicationStatus {
+	case replication.Pending:
+		sizeS.pendingSize += oi.Size
+		globalReplicationState.queueReplicaTask(oi)
+	case replication.Failed:
+		sizeS.failedSize += oi.Size
+		globalReplicationState.queueReplicaTask(oi)
+	case replication.Complete:
+		sizeS.replicatedSize += oi.Size
+	case replication.Replica:
+		sizeS.replicaSize += oi.Size
 	}
 }
 
 // healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, meta actionMeta) {
+func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
 	// handle soft delete and permanent delete failures here.
-	if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		versionID := ""
 		dmVersionID := ""
-		if meta.oi.VersionPurgeStatus.Empty() {
-			dmVersionID = meta.oi.VersionID
+		if oi.VersionPurgeStatus.Empty() {
+			dmVersionID = oi.VersionID
 		} else {
-			versionID = meta.oi.VersionID
+			versionID = oi.VersionID
 		}
 		globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
 			DeletedObject: DeletedObject{
-				ObjectName:                    meta.oi.Name,
+				ObjectName:                    oi.Name,
 				DeleteMarkerVersionID:         dmVersionID,
 				VersionID:                     versionID,
-				DeleteMarkerReplicationStatus: string(meta.oi.ReplicationStatus),
-				DeleteMarkerMTime:             DeleteMarkerMTime{meta.oi.ModTime},
-				DeleteMarker:                  meta.oi.DeleteMarker,
-				VersionPurgeStatus:            meta.oi.VersionPurgeStatus,
+				DeleteMarkerReplicationStatus: string(oi.ReplicationStatus),
+				DeleteMarkerMTime:             DeleteMarkerMTime{oi.ModTime},
+				DeleteMarker:                  oi.DeleteMarker,
+				VersionPurgeStatus:            oi.VersionPurgeStatus,
 			},
-			Bucket: meta.oi.Bucket,
+			Bucket: oi.Bucket,
 		})
 	}
 }
+
+type dynamicSleeper struct {
+	mu sync.RWMutex
+
+	// Sleep factor
+	factor float64
+
+	// maximum sleep cap,
+	// set to <= 0 to disable.
+	maxSleep time.Duration
+
+	// Don't sleep at all, if time taken is below this value.
+	// This is to avoid too small costly sleeps.
+	minSleep time.Duration
+
+	// cycle will be closed
+	cycle chan struct{}
+}
+
+// newDynamicSleeper
+func newDynamicSleeper(factor float64, maxWait time.Duration) *dynamicSleeper {
+	return &dynamicSleeper{
+		factor:   factor,
+		cycle:    make(chan struct{}),
+		maxSleep: maxWait,
+		minSleep: 100 * time.Microsecond,
+	}
+}
+
+// Timer returns a timer that has started.
+// When the returned function is called it will wait.
+func (d *dynamicSleeper) Timer(ctx context.Context) func() {
+	t := time.Now()
+	return func() {
+		doneAt := time.Now()
+		for {
+			// Grab current values
+			d.mu.RLock()
+			minWait, maxWait := d.minSleep, d.maxSleep
+			factor := d.factor
+			cycle := d.cycle
+			d.mu.RUnlock()
+			elapsed := doneAt.Sub(t)
+			// Don't sleep for really small amount of time
+			wantSleep := time.Duration(float64(elapsed) * factor)
+			if wantSleep <= minWait {
+				return
+			}
+			if maxWait > 0 && wantSleep > maxWait {
+				wantSleep = maxWait
+			}
+			timer := time.NewTimer(wantSleep)
+			select {
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			case <-timer.C:
+				return
+			case <-cycle:
+				if !timer.Stop() {
+					// We expired.
+					<-timer.C
+					return
+				}
+			}
+		}
+	}
+}
+
+// Sleep sleeps the specified time multiplied by the sleep factor.
+// If the factor is updated the sleep will be done again with the new factor.
+func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
+	for {
+		// Grab current values
+		d.mu.RLock()
+		minWait, maxWait := d.minSleep, d.maxSleep
+		factor := d.factor
+		cycle := d.cycle
+		d.mu.RUnlock()
+		// Don't sleep for really small amount of time
+		wantSleep := time.Duration(float64(base) * factor)
+		if wantSleep <= minWait {
+			return
+		}
+		if maxWait > 0 && wantSleep > maxWait {
+			wantSleep = maxWait
+		}
+		timer := time.NewTimer(wantSleep)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return
+		case <-timer.C:
+			return
+		case <-cycle:
+			if !timer.Stop() {
+				// We expired.
+				<-timer.C
+				return
+			}
+		}
+	}
+}
+
+// Update the current settings and cycle all waiting.
+// Parameters are the same as in the contructor.
+func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if math.Abs(d.factor-factor) < 1e-10 && d.maxSleep == maxWait {
+		return nil
+	}
+	// Update values and cycle waiting.
+	close(d.cycle)
+	d.factor = factor
+	d.maxSleep = maxWait
+	d.cycle = make(chan struct{})
+	return nil
+}

cmd/data-update-tracker.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
 	"github.com/willf/bloom"
 )
@@ -79,7 +80,7 @@ func newDataUpdateTracker() *dataUpdateTracker {
 		Current: dataUpdateFilter{
 			idx: 1,
 		},
-		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn,
+		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn || serverDebugLog,
 		input:      make(chan string, dataUpdateTrackerQueueSize),
 		save:       make(chan struct{}, 1),
 		saveExited: make(chan struct{}),
@@ -171,16 +172,17 @@ func (d *dataUpdateTracker) current() uint64 {
 // latestWithDir returns the highest index that contains the directory.
 // This means that any cycle higher than this does NOT contain the entry.
 func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
+	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
 	bucket, _ := path2BucketObjectWithBasePath("", dir)
 	if bucket == "" {
 		if d.debug && len(dir) > 0 {
-			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", dir)
+			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
 		}
 		return d.current()
 	}
 	if isReservedOrInvalidBucket(bucket, false) {
 		if d.debug {
-			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, dir)
+			console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, dir)
 		}
 		return d.current()
 	}
@@ -191,7 +193,7 @@ func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
 		return d.Current.idx
 	}
 	if d.debug {
-		logger.Info("current bloom does NOT contains dir %s", dir)
+		console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
 	}
 
 	idx := d.Current.idx - 1
@@ -281,7 +283,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
 		d.Saved = UTCNow()
 		err := d.serialize(&buf)
 		if d.debug {
-			logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx)
+			console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
 		}
 		d.dirty = false
 		d.mu.Unlock()
@@ -404,7 +406,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 	}
 	switch tmp[0] {
 	case 1, 2, 3:
-		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
+		console.Println(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		return nil
 	case dataUpdateTrackerVersion:
 	default:
@@ -481,14 +483,14 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 		bucket, _ := path2BucketObjectWithBasePath("", in)
 		if bucket == "" {
 			if d.debug && len(in) > 0 {
-				logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
+				console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
 			}
 			continue
 		}
 
 		if isReservedOrInvalidBucket(bucket, false) {
 			if d.debug {
-				logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+				console.Debugf(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
 			}
 			continue
 		}
@@ -498,7 +500,7 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 		d.mu.Lock()
 		for i := range split {
 			if d.debug {
-				logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+				console.Debugln(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
 			}
 			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
 		}
@@ -510,16 +512,17 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 // markDirty adds the supplied path to the current bloom filter.
 func (d *dataUpdateTracker) markDirty(in string) {
 	bucket, _ := path2BucketObjectWithBasePath("", in)
+	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
 	if bucket == "" {
 		if d.debug && len(in) > 0 {
-			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
+			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", in)
 		}
 		return
 	}
 
 	if isReservedOrInvalidBucket(bucket, false) {
 		if d.debug && false {
-			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+			console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
 		}
 		return
 	}
@@ -529,7 +532,7 @@ func (d *dataUpdateTracker) markDirty(in string) {
 	d.mu.Lock()
 	for i := range split {
 		if d.debug {
-			logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+			console.Debugln(dateUpdateTrackerLogPrefix + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
 		}
 		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
 	}
@@ -620,7 +623,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequ
 	if d.Current.idx != current {
 		d.dirty = true
 		if d.debug {
-			logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current)
+			console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
 		}
 
 		d.History = append(d.History, d.Current)

cmd/data-update-tracker_test.go

@@ -225,7 +225,7 @@ func TestDataUpdateTracker(t *testing.T) {
 
 	// Rerun test with returned bfr2
 	bf := dut.newBloomFilter()
-	_, err = bf.ReadFrom(bytes.NewBuffer(bfr2.Filter))
+	_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/data-usage-cache.go

@@ -17,11 +17,11 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"path"
 	"path/filepath"
 	"strings"
@@ -46,11 +46,14 @@ type sizeHistogram [dataUsageBucketLen]uint64
 //msgp:tuple dataUsageEntry
 type dataUsageEntry struct {
 	// These fields do no include any children.
-	Size     int64
-	Objects  uint64
-	ObjSizes sizeHistogram
-
-	Children dataUsageHashMap
+	Size                   int64
+	ReplicatedSize         uint64
+	ReplicationPendingSize uint64
+	ReplicationFailedSize  uint64
+	ReplicaSize            uint64
+	Objects                uint64
+	ObjSizes               sizeHistogram
+	Children               dataUsageHashMap
 }
 
 // dataUsageCache contains a cache of data usage entries.
@@ -76,10 +79,23 @@ type dataUsageCacheInfo struct {
 	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }
 
+func (e *dataUsageEntry) addSizes(summary sizeSummary) {
+	e.Size += summary.totalSize
+	e.ReplicatedSize += uint64(summary.replicatedSize)
+	e.ReplicationFailedSize += uint64(summary.failedSize)
+	e.ReplicationPendingSize += uint64(summary.pendingSize)
+	e.ReplicaSize += uint64(summary.replicaSize)
+}
+
 // merge other data usage entry into this, excluding children.
 func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Size += other.Size
+	e.ReplicationPendingSize += other.ReplicationPendingSize
+	e.ReplicationFailedSize += other.ReplicationFailedSize
+	e.ReplicatedSize += other.ReplicatedSize
+	e.ReplicaSize += other.ReplicaSize
+
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
 	}
@@ -213,11 +229,15 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	}
 	flat := d.flatten(*e)
 	return DataUsageInfo{
-		LastUpdate:        d.Info.LastUpdate,
-		ObjectsTotalCount: flat.Objects,
-		ObjectsTotalSize:  uint64(flat.Size),
-		BucketsCount:      uint64(len(e.Children)),
-		BucketsUsage:      d.bucketsUsageInfo(buckets),
+		LastUpdate:             d.Info.LastUpdate,
+		ObjectsTotalCount:      flat.Objects,
+		ObjectsTotalSize:       uint64(flat.Size),
+		ReplicatedSize:         flat.ReplicatedSize,
+		ReplicationFailedSize:  flat.ReplicationFailedSize,
+		ReplicationPendingSize: flat.ReplicationPendingSize,
+		ReplicaSize:            flat.ReplicaSize,
+		BucketsCount:           uint64(len(e.Children)),
+		BucketsUsage:           d.bucketsUsageInfo(buckets),
 	}
 }
 
@@ -343,9 +363,13 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		}
 		flat := d.flatten(*e)
 		dst[bucket.Name] = BucketUsageInfo{
-			Size:                 uint64(flat.Size),
-			ObjectsCount:         flat.Objects,
-			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+			Size:                   uint64(flat.Size),
+			ObjectsCount:           flat.Objects,
+			ReplicationPendingSize: flat.ReplicationPendingSize,
+			ReplicatedSize:         flat.ReplicatedSize,
+			ReplicationFailedSize:  flat.ReplicationFailedSize,
+			ReplicaSize:            flat.ReplicaSize,
+			ObjectSizesHistogram:   flat.ObjSizes.toMap(),
 		}
 	}
 	return dst
@@ -360,9 +384,13 @@ func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
 	}
 	flat := d.flatten(*e)
 	return BucketUsageInfo{
-		Size:                 uint64(flat.Size),
-		ObjectsCount:         flat.Objects,
-		ObjectSizesHistogram: flat.ObjSizes.toMap(),
+		Size:                   uint64(flat.Size),
+		ObjectsCount:           flat.Objects,
+		ReplicationPendingSize: flat.ReplicationPendingSize,
+		ReplicatedSize:         flat.ReplicatedSize,
+		ReplicationFailedSize:  flat.ReplicationFailedSize,
+		ReplicaSize:            flat.ReplicaSize,
+		ObjectSizesHistogram:   flat.ObjSizes.toMap(),
 	}
 }
 
@@ -430,7 +458,7 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 }
 
 type objectIO interface {
-	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
@@ -438,8 +466,7 @@ type objectIO interface {
 // Only backend errors are returned as errors.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
-	var buf bytes.Buffer
-	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
+	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		switch err.(type) {
 		case ObjectNotFound:
@@ -451,19 +478,23 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		*d = dataUsageCache{}
 		return nil
 	}
-	err = d.deserialize(&buf)
-	if err != nil {
+	defer r.Close()
+	if err := d.deserialize(r); err != nil {
 		*d = dataUsageCache{}
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, err.Error())
 	}
 	return nil
 }
 
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
-	b := d.serialize()
-	size := int64(len(b))
-	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+	pr, pw := io.Pipe()
+	go func() {
+		pw.CloseWithError(d.serializeTo(pw))
+	}()
+	defer pr.Close()
+
+	r, err := hash.NewReader(pr, -1, "", "", -1, false)
 	if err != nil {
 		return err
 	}
@@ -482,35 +513,36 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // dataUsageCacheVer indicates the cache version.
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
-const dataUsageCacheVer = 2
+const dataUsageCacheVer = 3
 
 // serialize the contents of the cache.
-func (d *dataUsageCache) serialize() []byte {
-	// Prepend version and compress.
-	dst := make([]byte, 0, d.Msgsize()+1)
-	dst = append(dst, dataUsageCacheVer)
-	buf := bytes.NewBuffer(dst)
-	enc, err := zstd.NewWriter(buf,
+func (d *dataUsageCache) serializeTo(dst io.Writer) error {
+	// Add version and compress.
+	_, err := dst.Write([]byte{dataUsageCacheVer})
+	if err != nil {
+		return err
+	}
+	enc, err := zstd.NewWriter(dst,
 		zstd.WithEncoderLevel(zstd.SpeedFastest),
 		zstd.WithWindowSize(1<<20),
 		zstd.WithEncoderConcurrency(2))
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
+		return err
 	}
 	mEnc := msgp.NewWriter(enc)
 	err = d.EncodeMsg(mEnc)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
+		return err
+	}
+	err = mEnc.Flush()
+	if err != nil {
+		return err
 	}
-	mEnc.Flush()
 	err = enc.Close()
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
+		return err
 	}
-	return buf.Bytes()
+	return nil
 }
 
 // deserialize the supplied byte slice into the cache.
@@ -521,7 +553,7 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		return io.ErrUnexpectedEOF
 	}
 	switch b[0] {
-	case 1:
+	case 1, 2:
 		return errors.New("cache version deprecated (will autoupdate)")
 	case dataUsageCacheVer:
 	default:

cmd/data-usage-cache_gen.go

@@ -492,8 +492,8 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 4 {
-		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+	if zb0001 != 8 {
+		err = msgp.ArrayError{Wanted: 8, Got: zb0001}
 		return
 	}
 	z.Size, err = dc.ReadInt64()
@@ -501,6 +501,26 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "Size")
 		return
 	}
+	z.ReplicatedSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	z.ReplicationPendingSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	z.ReplicationFailedSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	z.ReplicaSize, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
 	z.Objects, err = dc.ReadUint64()
 	if err != nil {
 		err = msgp.WrapError(err, "Objects")
@@ -533,8 +553,8 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 4
-	err = en.Append(0x94)
+	// array header, size 8
+	err = en.Append(0x98)
 	if err != nil {
 		return
 	}
@@ -543,6 +563,26 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Size")
 		return
 	}
+	err = en.WriteUint64(z.ReplicatedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicationPendingSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicationFailedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	err = en.WriteUint64(z.ReplicaSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
 	err = en.WriteUint64(z.Objects)
 	if err != nil {
 		err = msgp.WrapError(err, "Objects")
@@ -571,9 +611,13 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 4
-	o = append(o, 0x94)
+	// array header, size 8
+	o = append(o, 0x98)
 	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.ReplicatedSize)
+	o = msgp.AppendUint64(o, z.ReplicationPendingSize)
+	o = msgp.AppendUint64(o, z.ReplicationFailedSize)
+	o = msgp.AppendUint64(o, z.ReplicaSize)
 	o = msgp.AppendUint64(o, z.Objects)
 	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
 	for za0001 := range z.ObjSizes {
@@ -595,8 +639,8 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 4 {
-		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+	if zb0001 != 8 {
+		err = msgp.ArrayError{Wanted: 8, Got: zb0001}
 		return
 	}
 	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
@@ -604,6 +648,26 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "Size")
 		return
 	}
+	z.ReplicatedSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	z.ReplicationPendingSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationPendingSize")
+		return
+	}
+	z.ReplicationFailedSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationFailedSize")
+		return
+	}
+	z.ReplicaSize, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
 	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err, "Objects")
@@ -637,7 +701,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
 	return
 }
 

cmd/data-usage.go

@@ -27,8 +27,6 @@ import (
 )
 
 const (
-	envDataUsageCrawlConf  = "MINIO_DISK_USAGE_CRAWL_ENABLE"
-	envDataUsageCrawlDelay = "MINIO_DISK_USAGE_CRAWL_DELAY"
 	envDataUsageCrawlDebug = "MINIO_DISK_USAGE_CRAWL_DEBUG"
 
 	dataUsageRoot   = SlashSeparator
@@ -40,8 +38,8 @@ const (
 )
 
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
-	for dataUsageInfo := range gui {
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
+	for dataUsageInfo := range dui {
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
 		if err != nil {
 			logger.LogIf(ctx, err)
@@ -53,7 +51,6 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan
 			logger.LogIf(ctx, err)
 			continue
 		}
-
 		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
 		if !isErrBucketNotFound(err) {
 			logger.LogIf(ctx, err)

cmd/data-usage_test.go

@@ -51,15 +51,17 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item crawlItem) (i int64, err error) {
+	getSize := func(item crawlItem) (sizeS sizeSummary, err error) {
 		if item.Typ&os.ModeDir == 0 {
-			s, err := os.Stat(item.Path)
+			var s os.FileInfo
+			s, err = os.Stat(item.Path)
 			if err != nil {
-				return 0, err
+				return
 			}
-			return s.Size(), nil
+			sizeS.totalSize = s.Size()
+			return sizeS, nil
 		}
-		return 0, nil
+		return
 	}
 
 	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
@@ -345,15 +347,17 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	createUsageTestFiles(t, base, "", files)
 
-	getSize := func(item crawlItem) (i int64, err error) {
+	getSize := func(item crawlItem) (sizeS sizeSummary, err error) {
 		if item.Typ&os.ModeDir == 0 {
-			s, err := os.Stat(item.Path)
+			var s os.FileInfo
+			s, err = os.Stat(item.Path)
 			if err != nil {
-				return 0, err
+				return
 			}
-			return s.Size(), nil
+			sizeS.totalSize = s.Size()
+			return
 		}
-		return 0, nil
+		return
 	}
 	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
 	if err != nil {
@@ -642,28 +646,33 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 	}
 	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item crawlItem) (i int64, err error) {
+	getSize := func(item crawlItem) (sizeS sizeSummary, err error) {
 		if item.Typ&os.ModeDir == 0 {
-			s, err := os.Stat(item.Path)
+			var s os.FileInfo
+			s, err = os.Stat(item.Path)
 			if err != nil {
-				return 0, err
+				return
 			}
-			return s.Size(), nil
+			sizeS.totalSize = s.Size()
+			return
 		}
-		return 0, nil
+		return
 	}
 	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	b := want.serialize()
-	var got dataUsageCache
-	err = got.deserialize(bytes.NewBuffer(b))
+	var buf bytes.Buffer
+	err = want.serializeTo(&buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("serialized size:", buf.Len(), "bytes")
+	var got dataUsageCache
+	err = got.deserialize(&buf)
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Log("serialized size:", len(b), "bytes")
 	if got.Info.LastUpdate.IsZero() {
 		t.Error("lastupdate not set")
 	}

cmd/disk-cache-backend.go

@@ -962,7 +962,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	}
 	if globalCacheKMS != nil {
 		// clean up internal SSE cache metadata
-		delete(gr.ObjInfo.UserDefined, crypto.SSEHeader)
+		delete(gr.ObjInfo.UserDefined, xhttp.AmzServerSideEncryption)
 	}
 	if !rngInfo.Empty() {
 		// overlay Size with actual object size and not the range size

cmd/disk-cache-stats.go

@@ -23,6 +23,10 @@ import (
 // CacheDiskStats represents cache disk statistics
 // such as current disk usage and available.
 type CacheDiskStats struct {
+	// used cache size
+	UsageSize uint64
+	// total cache disk capacity
+	TotalCapacity uint64
 	// indicates if usage is high or low, if high value is '1', if low its '0'
 	UsageState int32
 	// indicates the current usage percentage of this cache disk

cmd/disk-cache.go

@@ -781,6 +781,10 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 			dcache := c.cache[i]
 			cacheDiskStats[i] = CacheDiskStats{}
 			if dcache != nil {
+				info, err := getDiskInfo(dcache.dir)
+				logger.LogIf(ctx, err)
+				cacheDiskStats[i].UsageSize = info.Used
+				cacheDiskStats[i].TotalCapacity = info.Total
 				cacheDiskStats[i].Dir = dcache.stats.Dir
 				atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
 				atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))

cmd/dummy-data-generator_test.go

@@ -163,8 +163,8 @@ func cmpReaders(r1, r2 io.Reader) (bool, string) {
 
 func TestCmpReaders(t *testing.T) {
 	{
-		r1 := bytes.NewBuffer([]byte("abc"))
-		r2 := bytes.NewBuffer([]byte("abc"))
+		r1 := bytes.NewReader([]byte("abc"))
+		r2 := bytes.NewReader([]byte("abc"))
 		ok, msg := cmpReaders(r1, r2)
 		if !(ok && msg == "") {
 			t.Fatalf("unexpected")
@@ -172,8 +172,8 @@ func TestCmpReaders(t *testing.T) {
 	}
 
 	{
-		r1 := bytes.NewBuffer([]byte("abc"))
-		r2 := bytes.NewBuffer([]byte("abcd"))
+		r1 := bytes.NewReader([]byte("abc"))
+		r2 := bytes.NewReader([]byte("abcd"))
 		ok, _ := cmpReaders(r1, r2)
 		if ok {
 			t.Fatalf("unexpected")

cmd/encryption-v1.go

@@ -353,9 +353,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 
 // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
 // reader
-func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
-	length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
-	io.Reader, error) {
+func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) {
 
 	bucket, object := oi.Bucket, oi.Name
 	// Single part case
@@ -386,13 +384,13 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 		header:            h,
 		bucket:            bucket,
 		object:            object,
-		customerKeyHeader: h.Get(crypto.SSECKey),
+		customerKeyHeader: h.Get(xhttp.AmzServerSideEncryptionCustomerKey),
 		copySource:        copySource,
 		metadata:          cloneMSS(oi.UserDefined),
 	}
 
 	if w.copySource {
-		w.customerKeyHeader = h.Get(crypto.SSECopyKey)
+		w.customerKeyHeader = h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey)
 	}
 
 	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
@@ -434,12 +432,12 @@ func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
 	var err error
 	if d.copySource {
 		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
+			d.header.Set(xhttp.AmzServerSideEncryptionCopyCustomerKey, d.customerKeyHeader)
 			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
 		}
 	} else {
 		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(crypto.SSECKey, d.customerKeyHeader)
+			d.header.Set(xhttp.AmzServerSideEncryptionCustomerKey, d.customerKeyHeader)
 			key, err = ParseSSECustomerHeader(d.header)
 		}
 	}

cmd/encryption-v1_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/sio"
 )
 
@@ -38,27 +39,27 @@ var encryptRequestTests = []struct {
 }{
 	{
 		header: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
+			xhttp.AmzServerSideEncryptionCustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{},
 	},
 	{
 		header: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
+			xhttp.AmzServerSideEncryptionCustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{
-			crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
 		},
 	},
 }
 
 func TestEncryptRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	globalIsSSL = true
+	defer func(flag bool) { globalIsTLS = flag }(globalIsTLS)
+	globalIsTLS = true
 	for i, test := range encryptRequestTests {
 		content := bytes.NewReader(make([]byte, 64))
 		req := &http.Request{Header: http.Header{}}
@@ -70,13 +71,13 @@ func TestEncryptRequest(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
-		if kdf, ok := test.metadata[crypto.SSESealAlgorithm]; !ok {
+		if kdf, ok := test.metadata[crypto.MetaAlgorithm]; !ok {
 			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
 		}
-		if iv, ok := test.metadata[crypto.SSEIV]; !ok {
+		if iv, ok := test.metadata[crypto.MetaIV]; !ok {
 			t.Errorf("Test %d: crypto.SSEIV must be part of metadata: %v", i, iv)
 		}
-		if mac, ok := test.metadata[crypto.SSECSealedKey]; !ok {
+		if mac, ok := test.metadata[crypto.MetaSealedKeySSEC]; !ok {
 			t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
 		}
 	}
@@ -93,33 +94,33 @@ var decryptObjectInfoTests = []struct {
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
-		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
+		request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
-		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
+		request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSECSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaSealedKeySSEC: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
 		request: &http.Request{Header: http.Header{}},
 		expErr:  errEncryptedObject,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
-		request: &http.Request{Method: http.MethodGet, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Method: http.MethodGet, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  errInvalidEncryptionParameters,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
-		request: &http.Request{Method: http.MethodHead, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		request: &http.Request{Method: http.MethodHead, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  errInvalidEncryptionParameters,
 	},
 	{
-		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
-		request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
+		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
+		request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
 		expErr:  errObjectTampered,
 	},
 }
@@ -221,10 +222,10 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
 		Name:   "object",
 		Size:   595160760,
 		UserDefined: map[string]string{
-			crypto.SSEMultipart:                    "",
-			crypto.SSEIV:                           "HTexa=",
-			crypto.SSESealAlgorithm:                "DAREv2-HMAC-SHA256",
-			crypto.SSECSealedKey:                   "IAA8PGAA==",
+			crypto.MetaMultipart:                   "",
+			crypto.MetaIV:                          "HTexa=",
+			crypto.MetaAlgorithm:                   "DAREv2-HMAC-SHA256",
+			crypto.MetaSealedKeySSEC:               "IAA8PGAA==",
 			ReservedMetadataPrefix + "actual-size": "594870264",
 			"content-type":                         "application/octet-stream",
 			"etag":                                 "166b1545b4c1535294ee0686678bea8c-2",
@@ -276,11 +277,11 @@ func TestGetDecryptedRange(t *testing.T) {
 		}
 		udMap = func(isMulti bool) map[string]string {
 			m := map[string]string{
-				crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
-				crypto.SSEMultipart:     "1",
+				crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm,
+				crypto.MetaMultipart: "1",
 			}
 			if !isMulti {
-				delete(m, crypto.SSEMultipart)
+				delete(m, crypto.MetaMultipart)
 			}
 			return m
 		}
@@ -553,56 +554,56 @@ var getDefaultOptsTests = []struct {
 	encryptionType encrypt.Type
 	err            error
 }{
-	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
-		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
+		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: encrypt.SSEC,
 		err:            nil}, // 0
-	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
-		crypto.SSECKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
+		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     true,
 		metadata:       nil,
 		encryptionType: "",
 		err:            nil}, // 1
-	{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
-		crypto.SSECKey:    []string{"Mz"},
-		crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
+		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"Mz"},
+		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: "",
 		err:            crypto.ErrInvalidCustomerKey}, // 2
-	{headers: http.Header{crypto.SSEHeader: []string{"AES256"}},
+	{headers: http.Header{xhttp.AmzServerSideEncryption: []string{"AES256"}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: encrypt.S3,
 		err:            nil}, // 3
 	{headers: http.Header{},
 		copySource: false,
-		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
-			crypto.S3KMSKeyID:     "kms-key",
-			crypto.S3KMSSealedKey: "m-key"},
+		metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			crypto.MetaKeyID:             "kms-key",
+			crypto.MetaDataEncryptionKey: "m-key"},
 		encryptionType: encrypt.S3,
 		err:            nil}, // 4
 	{headers: http.Header{},
 		copySource: true,
-		metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
-			crypto.S3KMSKeyID:     "kms-key",
-			crypto.S3KMSSealedKey: "m-key"},
+		metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+			crypto.MetaKeyID:             "kms-key",
+			crypto.MetaDataEncryptionKey: "m-key"},
 		encryptionType: "",
 		err:            nil}, // 5
-	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
-		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
+		xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     true,
 		metadata:       nil,
 		encryptionType: encrypt.SSEC,
 		err:            nil}, // 6
-	{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
-		crypto.SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+	{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
+		xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: "",

cmd/endpoint.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
@@ -38,7 +37,6 @@ import (
 	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
 	xnet "github.com/minio/minio/pkg/net"
@@ -59,7 +57,7 @@ const (
 // See proxyRequest() for details.
 type ProxyEndpoint struct {
 	Endpoint
-	Transport *http.Transport
+	Transport http.RoundTripper
 }
 
 // Endpoint - any type of endpoint.
@@ -198,7 +196,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}, nil
 }
 
-// ZoneEndpoints represent endpoints in a given zone
+// ZoneEndpoints represent endpoints in a given pool
 // along with its setCount and setDriveCount.
 type ZoneEndpoints struct {
 	SetCount     int
@@ -209,8 +207,8 @@ type ZoneEndpoints struct {
 // EndpointServerPools - list of list of endpoints
 type EndpointServerPools []ZoneEndpoints
 
-// GetLocalZoneIdx returns the zone which endpoint belongs to locally.
-// if ep is remote this code will return -1 zoneIndex
+// GetLocalZoneIdx returns the pool which endpoint belongs to locally.
+// if ep is remote this code will return -1 poolIndex
 func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
 	for i, zep := range l {
 		for _, cep := range zep.Endpoints {
@@ -224,7 +222,7 @@ func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
 	return -1
 }
 
-// Add add zone endpoints
+// Add add pool endpoints
 func (l *EndpointServerPools) Add(zeps ZoneEndpoints) error {
 	existSet := set.NewStringSet()
 	for _, zep := range *l {
@@ -242,6 +240,22 @@ func (l *EndpointServerPools) Add(zeps ZoneEndpoints) error {
 	return nil
 }
 
+// Localhost - returns the local hostname from list of endpoints
+func (l EndpointServerPools) Localhost() string {
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				u := &url.URL{
+					Scheme: endpoint.Scheme,
+					Host:   endpoint.Host,
+				}
+				return u.String()
+			}
+		}
+	}
+	return ""
+}
+
 // FirstLocal returns true if the first endpoint is local.
 func (l EndpointServerPools) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
@@ -337,6 +351,14 @@ func (endpoints Endpoints) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// GetAllStrings - returns allstring of all endpoints
+func (endpoints Endpoints) GetAllStrings() (all []string) {
+	for _, e := range endpoints {
+		all = append(all, e.String())
+	}
+	return
+}
+
 func hostResolveToLocalhost(endpoint Endpoint) bool {
 	hostIPs, err := getHostIP(endpoint.Hostname())
 	if err != nil {
@@ -456,8 +478,8 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 						// participate atleast one disk and be local.
 						//
 						// In special cases for replica set with expanded
-						// zone setups we need to make sure to provide
-						// value of foundPrevLocal from zone1 if we already
+						// pool setups we need to make sure to provide
+						// value of foundPrevLocal from pool1 if we already
 						// found a local setup. Only if we haven't found
 						// previous local we continue to wait to look for
 						// atleast one local.
@@ -857,20 +879,9 @@ func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint
 			}
 			proxyEpSet.Add(host)
 
-			var tlsConfig *tls.Config
-			if globalIsSSL {
-				tlsConfig = &tls.Config{
-					ServerName: endpoint.Hostname(),
-					RootCAs:    globalRootCAs,
-				}
-			}
-
-			// allow transport to be HTTP/1.1 for proxying.
-			tr := newCustomHTTPProxyTransport(tlsConfig, rest.DefaultTimeout)()
-
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,
-				Transport: tr,
+				Transport: globalProxyTransport,
 			})
 		}
 	}

cmd/erasure-common.go

@@ -18,7 +18,8 @@ package cmd
 
 import (
 	"context"
-	"path"
+	"sort"
+	"strings"
 	"sync"
 
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -37,6 +38,65 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
+type sortSlices struct {
+	disks []StorageAPI
+	infos []DiskInfo
+}
+
+type sortByOther sortSlices
+
+func (sbo sortByOther) Len() int {
+	return len(sbo.disks)
+}
+
+func (sbo sortByOther) Swap(i, j int) {
+	sbo.disks[i], sbo.disks[j] = sbo.disks[j], sbo.disks[i]
+	sbo.infos[i], sbo.infos[j] = sbo.infos[j], sbo.infos[i]
+}
+
+func (sbo sortByOther) Less(i, j int) bool {
+	return sbo.infos[i].UsedInodes < sbo.infos[j].UsedInodes
+}
+
+func (er erasureObjects) getOnlineDisksSortedByUsedInodes() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var infos []DiskInfo
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			mu.Lock()
+			newDisks = append(newDisks, disks[i-1])
+			infos = append(infos, di)
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+
+	slices := sortSlices{newDisks, infos}
+	sort.Sort(sortByOther(slices))
+
+	return slices.disks
+}
+
 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	var wg sync.WaitGroup
@@ -147,24 +207,29 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 // object is "a/b/c/d", stat makes sure that objects ""a/b/c""
 // "a/b" and "a" do not exist.
 func (er erasureObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
-	var isParentDirObject func(string) bool
-	isParentDirObject = func(p string) bool {
-		if p == "." || p == SlashSeparator {
+	path := ""
+	segments := strings.Split(parent, slashSeparator)
+	for _, s := range segments {
+		if s == "" {
+			break
+		}
+		path += s
+		isObject, pathNotExist := er.isObject(ctx, bucket, path)
+		if pathNotExist {
 			return false
 		}
-		if er.isObject(ctx, bucket, p) {
+		if isObject {
 			// If there is already a file at prefix "p", return true.
 			return true
 		}
-		// Check if there is a file as one of the parent paths.
-		return isParentDirObject(path.Dir(p))
+		path += slashSeparator
 	}
-	return isParentDirObject(parent)
+	return false
 }
 
 // isObject - returns `true` if the prefix is an object i.e if
 // `xl.meta` exists at the leaf, false otherwise.
-func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (ok bool) {
+func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (ok, pathDoesNotExist bool) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -186,5 +251,6 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
 	// ignored if necessary.
 	readQuorum := getReadQuorum(len(storageDisks))
 
-	return reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum) == nil
+	err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum)
+	return err == nil, err == errPathNotFound
 }

cmd/erasure-common_test.go

@@ -66,6 +66,10 @@ func TestErasureParentDirIsObject(t *testing.T) {
 			parentIsObject: true,
 			objectName:     objectName,
 		},
+		{
+			parentIsObject: false,
+			objectName:     "new-dir",
+		},
 		{
 			parentIsObject: false,
 			objectName:     "",

cmd/erasure-decode.go

@@ -33,6 +33,7 @@ type parallelReader struct {
 	readers       []io.ReaderAt
 	orgReaders    []io.ReaderAt
 	dataBlocks    int
+	errs          []error
 	offset        int64
 	shardSize     int64
 	shardFileSize int64
@@ -49,6 +50,7 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
 	return &parallelReader{
 		readers:       readers,
 		orgReaders:    readers,
+		errs:          make([]error, len(readers)),
 		dataBlocks:    e.dataBlocks,
 		offset:        (offset / e.blockSize) * e.ShardSize(),
 		shardSize:     e.ShardSize(),
@@ -169,6 +171,8 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 				// This will be communicated upstream.
 				p.orgReaders[bufIdx] = nil
 				p.readers[i] = nil
+				p.errs[i] = err
+
 				// Since ReadAt returned error, trigger another read.
 				readTriggerCh <- true
 				return
@@ -182,7 +186,6 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		readerIndex++
 	}
 	wg.Wait()
-
 	if p.canDecode(newBuf) {
 		p.offset += p.shardSize
 		if healRequired != 0 {
@@ -191,7 +194,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		return newBuf, nil
 	}
 
-	return nil, errErasureReadQuorum
+	return nil, reduceReadQuorumErrs(context.Background(), p.errs, objectOpIgnoredErrs, p.dataBlocks)
 }
 
 type errDecodeHealRequired struct {

cmd/erasure-decode_test.go

@@ -134,7 +134,7 @@ func TestErasureDecode(t *testing.T) {
 			}
 			tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
 
-			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
+			bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 		}
 
 		writer := bytes.NewBuffer(nil)
@@ -164,7 +164,7 @@ func TestErasureDecode(t *testing.T) {
 					continue
 				}
 				tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
-				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
+				bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
 				if bitrotReaders[j] == nil {
@@ -270,7 +270,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 				continue
 			}
 			tillOffset := erasure.ShardFileOffset(offset, readLen, length)
-			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
+			bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
 		closeBitrotReaders(bitrotReaders)
@@ -332,7 +332,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 				continue
 			}
 			tillOffset := erasure.ShardFileOffset(0, size, size)
-			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
+			bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {
 			panic(err)

cmd/erasure-heal_test.go

@@ -99,7 +99,7 @@ func TestErasureHeal(t *testing.T) {
 		readers := make([]io.ReaderAt, len(disks))
 		for i, disk := range disks {
 			shardFilesize := erasure.ShardFileSize(test.size)
-			readers[i] = newBitrotReader(disk, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
+			readers[i] = newBitrotReader(disk, nil, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
 		}
 
 		// setup stale disks for the test case

cmd/erasure-healing-common_test.go

@@ -186,7 +186,7 @@ func TestListOnlineDisks(t *testing.T) {
 			t.Fatalf("Failed to putObject %v", err)
 		}
 
-		partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+		partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 		fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
 		if err != nil {
 			t.Fatalf("Failed to getLatestFileInfo %v", err)
@@ -287,7 +287,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	readQuorum := len(erasureDisks) / 2
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		t.Fatalf("Failed to read xl meta data %v", reducedErr)
@@ -295,7 +295,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	// Test that all disks are returned without any failures with
 	// unmodified meta data
-	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	if err != nil {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}

cmd/erasure-healing.go

@@ -32,9 +32,9 @@ import (
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
-func (er erasureObjects) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (
+func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	result madmin.HealResultItem, err error) {
-	if !dryRun {
+	if !opts.DryRun {
 		defer ObjectPathUpdated(bucket)
 	}
 
@@ -45,12 +45,12 @@ func (er erasureObjects) HealBucket(ctx context.Context, bucket string, dryRun,
 	writeQuorum := getWriteQuorum(len(storageDisks))
 
 	// Heal bucket.
-	return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, dryRun)
+	return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, opts)
 }
 
 // Heal bucket - create buckets on disks where it does not exist.
 func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []string, bucket string, writeQuorum int,
-	dryRun bool) (res madmin.HealResultItem, err error) {
+	opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
 
 	// Initialize sync waitgroup.
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -84,7 +84,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 				afterState[index] = madmin.DriveStateMissing
 
 				// mutate only if not a dry-run
-				if dryRun {
+				if opts.DryRun {
 					return nil
 				}
 
@@ -99,7 +99,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 	errs := g.Wait()
 
 	reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum-1)
-	if reducedErr == errVolumeNotFound {
+	if errors.Is(reducedErr, errVolumeNotFound) && !opts.Recreate {
 		return res, nil
 	}
 
@@ -395,7 +395,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				if latestMeta.XLV1 {
 					partPath = pathJoin(object, fmt.Sprintf("part.%d", partNumber))
 				}
-				readers[i] = newBitrotReader(disk, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
+				readers[i] = newBitrotReader(disk, nil, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
 			}
 			writers := make([]io.Writer, len(outDatedDisks))
 			for i, disk := range outDatedDisks {
@@ -811,7 +811,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	storageEndpoints := er.getEndpoints()
 
 	// Read metadata files from all the disks
-	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID)
+	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
 
 	if isAllNotFound(errs) {
 		err = toObjectErr(errFileNotFound, bucket, object)

cmd/erasure-healing_test.go

@@ -130,7 +130,10 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 	// This would create the bucket.
-	_, err = er.HealBucket(ctx, bucket, false, false)
+	_, err = er.HealBucket(ctx, bucket, madmin.HealOpts{
+		DryRun: false,
+		Remove: false,
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -211,7 +214,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -236,7 +239,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -262,7 +265,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)

cmd/erasure-metadata-utils.go

@@ -113,21 +113,9 @@ func hashOrder(key string, cardinality int) []int {
 	return nums
 }
 
-// Reads all `xl.meta` metadata as a FileInfo slice and checks if the data dir exists as well,
-// otherwise returns errFileNotFound (or errFileVersionNotFound)
-func getAllObjectFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
-	return readVersionFromDisks(ctx, disks, bucket, object, versionID, true)
-}
-
 // Reads all `xl.meta` metadata as a FileInfo slice.
 // Returns error slice indicating the failed metadata reads.
-func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
-	return readVersionFromDisks(ctx, disks, bucket, object, versionID, false)
-}
-
-// Reads all `xl.meta` metadata as a FileInfo slice and checks if the data dir
-// exists as well, if checkDataDir is set to true.
-func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, checkDataDir bool) ([]FileInfo, []error) {
+func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, readData bool) ([]FileInfo, []error) {
 	metadataArray := make([]FileInfo, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
@@ -138,11 +126,10 @@ func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, objec
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, checkDataDir)
+			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, readData)
 			if err != nil {
-				if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
-					logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())
-					logger.LogIf(ctx, err)
+				if !IsErr(err, errFileNotFound, errVolumeNotFound, errFileVersionNotFound, errDiskNotFound) {
+					logger.LogOnceIf(ctx, err, disks[index].String())
 				}
 			}
 			return err

cmd/erasure-multipart.go

@@ -46,7 +46,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	disks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "")
+	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "", false)
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -89,27 +89,12 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (er erasureObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	ticker := time.NewTicker(cleanupInterval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			var disk StorageAPI
-			// run multiple cleanup's local to this server.
-			for _, d := range er.getLoadBalancedLocalDisks() {
-				if d != nil {
-					disk = d
-					break
-				}
-			}
-			if disk == nil {
-				continue
-			}
+func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
+	// run multiple cleanup's local to this server.
+	for _, disk := range er.getLoadBalancedLocalDisks() {
+		if disk != nil {
 			er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
+			return
 		}
 	}
 }
@@ -382,9 +367,11 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
+	storageDisks := er.getDisks()
+
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs = readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket,
-		uploadIDPath, "")
+	partsMetadata, errs = readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket,
+		uploadIDPath, "", false)
 
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -398,7 +385,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	}
 
 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(er.getDisks(), partsMetadata, errs)
+	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Pick one from the first valid metadata.
 	fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, writeQuorum)
@@ -487,7 +474,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	}
 
 	// Read metadata again because it might be updated with parallel upload of another part.
-	partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 	reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errErasureWriteQuorum {
 		return pi, toObjectErr(reducedErr, bucket, object)
@@ -565,7 +552,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID)
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID, false)
 
 	// get Quorum for this object
 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -613,7 +600,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -717,7 +704,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -859,21 +846,21 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Check if there is any offline disk and add it to the MRF list
-	for i, disk := range onlineDisks {
-		if disk == nil || storageDisks[i] == nil {
-			er.addPartial(bucket, object, fi.VersionID)
-			break
+	for _, disk := range onlineDisks {
+		if disk != nil && disk.IsOnline() {
+			continue
 		}
+		er.addPartial(bucket, object, fi.VersionID)
+		break
 	}
 
 	for i := 0; i < len(onlineDisks); i++ {
-		if onlineDisks[i] == nil {
-			continue
+		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+			// Object info is the same in all disks, so we can pick
+			// the first meta from online disk
+			fi = partsMetadata[i]
+			break
 		}
-		// Object info is the same in all disks, so we can pick
-		// the first meta from online disk
-		fi = partsMetadata[i]
-		break
 	}
 
 	// Success, return object info.
@@ -902,7 +889,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)

cmd/erasure-object.go

@@ -58,7 +58,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 
 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
-	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, false)
 
 	// get Quorum for this object
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
@@ -157,7 +157,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		unlockOnDefer = true
 	}
 
-	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
+	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
 		return nil, toObjectErr(err, bucket, object)
 	}
@@ -298,7 +298,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			}
 			checksumInfo := metaArr[index].Erasure.GetChecksumInfo(partNumber)
 			partPath := pathJoin(object, metaArr[index].DataDir, fmt.Sprintf("part.%d", partNumber))
-			readers[index] = newBitrotReader(disk, bucket, partPath, tillOffset,
+			data := metaArr[index].Data
+			readers[index] = newBitrotReader(disk, data, bucket, partPath, tillOffset,
 				checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
 
 			// Prefer local disks
@@ -337,7 +338,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 
 // getObject wrapper for erasure GetObject
 func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset, length int64, writer io.Writer, opts ObjectOptions) error {
-	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
+	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}
@@ -364,11 +365,11 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
 
-func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
+func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
 	disks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := getAllObjectFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -410,7 +411,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts)
+	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
@@ -608,7 +609,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	// Initialize parts metadata
-	partsMetadata := make([]FileInfo, len(er.getDisks()))
+	partsMetadata := make([]FileInfo, len(storageDisks))
 
 	fi := newFileInfo(object, dataDrives, parityDrives)
 
@@ -674,11 +675,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
-	lk := er.NewNSLock(bucket, object)
-	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
+	if !opts.NoLock {
+		lk := er.NewNSLock(bucket, object)
+		if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
+			return ObjectInfo{}, err
+		}
+		defer lk.Unlock()
 	}
-	defer lk.Unlock()
 
 	for i, w := range writers {
 		if w == nil {
@@ -727,20 +730,20 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// Whether a disk was initially or becomes offline
 	// during this upload, send it to the MRF list.
 	for i := 0; i < len(onlineDisks); i++ {
-		if onlineDisks[i] == nil || storageDisks[i] == nil {
-			er.addPartial(bucket, object, fi.VersionID)
-			break
+		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+			continue
 		}
+		er.addPartial(bucket, object, fi.VersionID)
+		break
 	}
 
 	for i := 0; i < len(onlineDisks); i++ {
-		if onlineDisks[i] == nil {
-			continue
+		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+			// Object info is the same in all disks, so we can pick
+			// the first meta from online disk
+			fi = partsMetadata[i]
+			break
 		}
-		// Object info is the same in all disks, so we can pick
-		// the first meta from online disk
-		fi = partsMetadata[i]
-		break
 	}
 
 	return fi.ToObjectInfo(bucket, object), nil
@@ -767,19 +770,18 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 // all the disks in parallel, including `xl.meta` associated with the
 // object.
 func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string, writeQuorum int) error {
-	var disks []StorageAPI
 	var err error
 	defer ObjectPathUpdated(pathJoin(bucket, object))
 
 	tmpObj := mustGetUUID()
+	disks := er.getDisks()
 	if bucket == minioMetaTmpBucket {
 		tmpObj = object
-		disks = er.getDisks()
 	} else {
 		// Rename the current object while requiring write quorum, but also consider
 		// that a non found object in a given disk as a success since it already
 		// confirms that the object doesn't have a part in that disk (already removed)
-		disks, err = rename(ctx, er.getDisks(), bucket, object, minioMetaTmpBucket, tmpObj, true, writeQuorum,
+		disks, err = rename(ctx, disks, bucket, object, minioMetaTmpBucket, tmpObj, true, writeQuorum,
 			[]error{errFileNotFound})
 		if err != nil {
 			return toObjectErr(err, bucket, object)
@@ -787,7 +789,6 @@ func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string
 	}
 
 	g := errgroup.WithNErrs(len(disks))
-
 	for index := range disks {
 		index := index
 		g.Go(func() error {
@@ -924,16 +925,15 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	for _, version := range versions {
 		// Check if there is any offline disk and add it to the MRF list
 		for _, disk := range storageDisks {
-			if disk == nil {
-				// ignore delete markers for quorum
-				if version.Deleted {
-					continue
-				}
-				// all other direct versionId references we should
-				// ensure no dangling file is left over.
-				er.addPartial(bucket, version.Name, version.VersionID)
-				break
+			if disk != nil && disk.IsOnline() {
+				// Skip attempted heal on online disks.
+				continue
 			}
+
+			// all other direct versionId references we should
+			// ensure no dangling file is left over.
+			er.addPartial(bucket, version.Name, version.VersionID)
+			break
 		}
 	}
 
@@ -945,6 +945,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	versionFound := true
+	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
 	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
 		switch gerr.(type) {
@@ -1043,10 +1044,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	for _, disk := range storageDisks {
-		if disk == nil {
-			er.addPartial(bucket, object, opts.VersionID)
-			break
+		if disk != nil && disk.IsOnline() {
+			continue
 		}
+		er.addPartial(bucket, object, opts.VersionID)
+		break
 	}
 
 	return ObjectInfo{
@@ -1072,7 +1074,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	disks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -1133,7 +1135,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	disks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {

cmd/erasure-object_test.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	crand "crypto/rand"
 	"errors"
+	"io"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -288,16 +290,42 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	bucket := "bucket"
 	object := "object"
 	opts := ObjectOptions{}
-	// Create "object" under "bucket".
-	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
+	buf := make([]byte, 33<<10)
+	if _, err = io.ReadFull(crand.Reader, buf); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test use case 1: All disks are online, xl.meta are present, but data are missing
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Make 9 disks offline, which leaves less than quorum number of disks
+	for _, disk := range xl.getDisks() {
+		files, _ := disk.ListDir(ctx, bucket, object, -1)
+		for _, file := range files {
+			if file != "xl.meta" {
+				disk.Delete(ctx, bucket, pathJoin(object, file), true)
+			}
+		}
+	}
+
+	err = xl.GetObject(ctx, bucket, object, 0, int64(len(buf)), ioutil.Discard, "", opts)
+	if err != toObjectErr(errFileNotFound, bucket, object) {
+		t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
+	}
+
+	// Test use case 2: Make 9 disks offline, which leaves less than quorum number of disks
 	// in a 16 disk Erasure setup. The original disks are 'replaced' with
 	// naughtyDisks that fail after 'f' successful StorageAPI method
 	// invocations, where f - [0,2)
+
+	// Create "object" under "bucket".
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	for f := 0; f < 2; f++ {
 		diskErrors := make(map[int]error)
 		for i := 0; i <= f; i++ {
@@ -320,9 +348,84 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		// Fetch object from store.
 		err = xl.GetObject(ctx, bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
 		if err != toObjectErr(errErasureReadQuorum, bucket, object) {
-			t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
+			t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
 		}
 	}
+
+}
+
+func TestHeadObjectNoQuorum(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Create an instance of xl backend.
+	obj, fsDirs, err := prepareErasure16(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Cleanup backend directories.
+	defer obj.Shutdown(context.Background())
+	defer removeRoots(fsDirs)
+
+	z := obj.(*erasureServerPools)
+	xl := z.serverPools[0].sets[0]
+
+	// Create "bucket"
+	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+	opts := ObjectOptions{}
+
+	// Test use case 1: All disks are online, xl.meta are present, but data are missing
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, disk := range xl.getDisks() {
+		files, _ := disk.ListDir(ctx, bucket, object, -1)
+		for _, file := range files {
+			if file != "xl.meta" {
+				disk.Delete(ctx, bucket, pathJoin(object, file), true)
+			}
+		}
+	}
+
+	_, err = xl.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		t.Errorf("Expected StatObject to succeed if data dir are not found, but failed with %v", err)
+	}
+
+	// Test use case 2: Make 9 disks offline, which leaves less than quorum number of disks
+	// in a 16 disk Erasure setup. The original disks are 'replaced' with
+	// naughtyDisks that fail after 'f' successful StorageAPI method
+	// invocations, where f - [0,2)
+
+	// Create "object" under "bucket".
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	erasureDisks := xl.getDisks()
+	for i := range erasureDisks[:10] {
+		erasureDisks[i] = nil
+	}
+
+	z.serverPools[0].erasureDisksMu.Lock()
+	xl.getDisks = func() []StorageAPI {
+		return erasureDisks
+	}
+	z.serverPools[0].erasureDisksMu.Unlock()
+
+	// Fetch object from store.
+	_, err = xl.GetObjectInfo(ctx, bucket, object, opts)
+	if err != toObjectErr(errErasureReadQuorum, bucket, object) {
+		t.Errorf("Expected getObjectInfo to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
+	}
 }
 
 func TestPutObjectNoQuorum(t *testing.T) {
@@ -424,7 +527,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "")
+	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "", false)
 
 	parts1SC := globalStorageClass
 
@@ -437,7 +540,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "")
+	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "", false)
 	parts2SC := globalStorageClass
 
 	// Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class
@@ -449,7 +552,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "")
+	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "", false)
 	parts3SC := globalStorageClass
 
 	// Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class
@@ -467,7 +570,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "")
+	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "", false)
 	parts4SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 6,
@@ -490,7 +593,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "")
+	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false)
 	parts5SC := storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
@@ -512,7 +615,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "")
+	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "", false)
 	parts6SC := storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
@@ -535,7 +638,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "")
+	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "", false)
 	parts7SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,

cmd/erasure-server-sets.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,6 +33,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -50,7 +52,7 @@ func (z *erasureServerPools) SingleZone() bool {
 	return len(z.serverPools) == 1
 }
 
-// Initialize new zone of erasure sets.
+// Initialize new pool of erasure sets.
 func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
 	var (
 		deploymentID string
@@ -99,12 +101,18 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 		idMap[id] = struct{}{}
 	}
 	res := make([]StorageAPI, 0, len(idMap))
-	for _, ss := range z.serverPools {
-		for _, disks := range ss.erasureDisks {
+	for _, s := range z.serverPools {
+		s.erasureDisksMu.RLock()
+		defer s.erasureDisksMu.RUnlock()
+		for _, disks := range s.erasureDisks {
 			for _, disk := range disks {
-				id, _ := disk.GetDiskID()
-				if _, ok := idMap[id]; ok {
-					res = append(res, disk)
+				if disk == OfflineDisk {
+					continue
+				}
+				if id, _ := disk.GetDiskID(); id != "" {
+					if _, ok := idMap[id]; ok {
+						res = append(res, disk)
+					}
 				}
 			}
 		}
@@ -120,9 +128,9 @@ func (z *erasureServerPools) SetDriveCount() int {
 	return z.serverPools[0].SetDriveCount()
 }
 
-type serverPoolsAvailableSpace []zoneAvailableSpace
+type serverPoolsAvailableSpace []poolAvailableSpace
 
-type zoneAvailableSpace struct {
+type poolAvailableSpace struct {
 	Index     int
 	Available uint64
 }
@@ -147,10 +155,10 @@ func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64
 	// choose when we reach this many
 	choose := rand.Uint64() % total
 	atTotal := uint64(0)
-	for _, zone := range serverPools {
-		atTotal += zone.Available
-		if atTotal > choose && zone.Available > 0 {
-			return zone.Index
+	for _, pool := range serverPools {
+		atTotal += pool.Available
+		if atTotal > choose && pool.Available > 0 {
+			return pool.Index
 		}
 	}
 	// Should not happen, but print values just in case.
@@ -158,8 +166,8 @@ func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64
 	return -1
 }
 
-// getServerPoolsAvailableSpace will return the available space of each zone after storing the content.
-// If there is not enough space the zone will return 0 bytes available.
+// getServerPoolsAvailableSpace will return the available space of each pool after storing the content.
+// If there is not enough space the pool will return 0 bytes available.
 // Negative sizes are seen as 0 bytes.
 func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, size int64) serverPoolsAvailableSpace {
 	if size < 0 {
@@ -201,7 +209,7 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
 				available = 0
 			}
 		}
-		serverPools[i] = zoneAvailableSpace{
+		serverPools[i] = poolAvailableSpace{
 			Index:     i,
 			Available: available,
 		}
@@ -209,14 +217,14 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
 	return serverPools
 }
 
-// getZoneIdx returns the found previous object and its corresponding zone idx,
-// if none are found falls back to most available space zone.
+// getZoneIdx returns the found previous object and its corresponding pool idx,
+// if none are found falls back to most available space pool.
 func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object string, opts ObjectOptions, size int64) (idx int, err error) {
 	if z.SingleZone() {
 		return 0, nil
 	}
-	for i, zone := range z.serverPools {
-		objInfo, err := zone.GetObjectInfo(ctx, bucket, object, opts)
+	for i, pool := range z.serverPools {
+		objInfo, err := pool.GetObjectInfo(ctx, bucket, object, opts)
 		switch err.(type) {
 		case ObjectNotFound:
 			// VersionId was not specified but found delete marker or no versions exist.
@@ -229,7 +237,7 @@ func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object stri
 			}
 		}
 		// delete marker not specified means no versions
-		// exist continue to next zone.
+		// exist continue to next pool.
 		if !objInfo.DeleteMarker && err != nil {
 			continue
 		}
@@ -266,9 +274,24 @@ func (z *erasureServerPools) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
+func (z *erasureServerPools) BackendInfo() (b BackendInfo) {
+	b.Type = BackendErasure
+
+	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
+	if scParity == 0 {
+		scParity = z.SetDriveCount() / 2
+	}
+	b.StandardSCData = z.SetDriveCount() - scParity
+	b.StandardSCParity = scParity
+
+	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
+	b.RRSCData = z.SetDriveCount() - rrSCParity
+	b.RRSCParity = rrSCParity
+	return
+}
+
+func (z *erasureServerPools) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 	var storageInfo StorageInfo
-	storageInfo.Backend.Type = BackendErasure
 
 	storageInfos := make([]StorageInfo, len(z.serverPools))
 	storageInfosErrs := make([][]error, len(z.serverPools))
@@ -276,7 +299,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (Stora
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
-			storageInfos[index], storageInfosErrs[index] = z.serverPools[index].StorageInfo(ctx, local)
+			storageInfos[index], storageInfosErrs[index] = z.serverPools[index].StorageInfo(ctx)
 			return nil
 		}, index)
 	}
@@ -284,23 +307,11 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (Stora
 	// Wait for the go routines.
 	g.Wait()
 
+	storageInfo.Backend = z.BackendInfo()
 	for _, lstorageInfo := range storageInfos {
 		storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
-		storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
-		storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
 	}
 
-	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	if scParity == 0 {
-		scParity = z.SetDriveCount() / 2
-	}
-
-	storageInfo.Backend.StandardSCData = z.SetDriveCount() - scParity
-	storageInfo.Backend.StandardSCParity = scParity
-	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
-	storageInfo.Backend.RRSCData = z.SetDriveCount() - rrSCParity
-	storageInfo.Backend.RRSCParity = rrSCParity
-
 	var errs []error
 	for i := range z.serverPools {
 		errs = append(errs, storageInfosErrs[i]...)
@@ -316,23 +327,25 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 	var mu sync.Mutex
 	var results []dataUsageCache
 	var firstErr error
-	var knownBuckets = make(map[string]struct{}) // used to deduplicate buckets.
-	var allBuckets []BucketInfo
+
+	allBuckets, err := z.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	if len(allBuckets) == 0 {
+		logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
+		updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state
+		return nil
+	}
+
+	// Crawl latest allBuckets first.
+	sort.Slice(allBuckets, func(i, j int) bool {
+		return allBuckets[i].Created.After(allBuckets[j].Created)
+	})
 
 	// Collect for each set in serverPools.
 	for _, z := range z.serverPools {
-		buckets, err := z.ListBuckets(ctx)
-		if err != nil {
-			return err
-		}
-		// Add new buckets.
-		for _, b := range buckets {
-			if _, ok := knownBuckets[b.Name]; ok {
-				continue
-			}
-			allBuckets = append(allBuckets, b)
-			knownBuckets[b.Name] = struct{}{}
-		}
 		for _, erObj := range z.sets {
 			wg.Add(1)
 			results = append(results, dataUsageCache{})
@@ -349,7 +362,7 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 					}
 				}()
 				// Start crawler. Blocks until done.
-				err := erObj.crawlAndGetDataUsage(ctx, buckets, bf, updates)
+				err := erObj.crawlAndGetDataUsage(ctx, allBuckets, bf, updates)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -370,7 +383,7 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 		defer updateTicker.Stop()
 		var lastUpdate time.Time
 
-		// We need to merge since we will get the same buckets from each zone.
+		// We need to merge since we will get the same buckets from each pool.
 		// Therefore to get the exact bucket sizes we must merge before we can convert.
 		var allMerged dataUsageCache
 
@@ -471,8 +484,8 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
 
 	object = encodeDirObject(object)
 
-	for _, zone := range z.serverPools {
-		gr, err = zone.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	for _, pool := range z.serverPools {
+		gr, err = pool.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -494,8 +507,8 @@ func (z *erasureServerPools) GetObject(ctx context.Context, bucket, object strin
 
 	object = encodeDirObject(object)
 
-	for _, zone := range z.serverPools {
-		if err := zone.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
+	for _, pool := range z.serverPools {
+		if err := pool.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
 			}
@@ -515,8 +528,8 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 	}
 
 	object = encodeDirObject(object)
-	for _, zone := range z.serverPools {
-		objInfo, err = zone.GetObjectInfo(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		objInfo, err = pool.GetObjectInfo(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -532,7 +545,7 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 	return objInfo, ObjectNotFound{Bucket: bucket, Object: object}
 }
 
-// PutObject - writes an object to least used erasure zone.
+// PutObject - writes an object to least used erasure pool.
 func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
 	// Validate put object input args.
 	if err := checkPutObjectArgs(ctx, bucket, object, z); err != nil {
@@ -550,7 +563,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		return ObjectInfo{}, err
 	}
 
-	// Overwrite the object at the right zone
+	// Overwrite the object at the right pool
 	return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
 }
 
@@ -564,8 +577,8 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	if z.SingleZone() {
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
-	for _, zone := range z.serverPools {
-		objInfo, err = zone.DeleteObject(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		objInfo, err = pool.DeleteObject(ctx, bucket, object, opts)
 		if err == nil {
 			return objInfo, nil
 		}
@@ -602,8 +615,8 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
 	}
 
-	for _, zone := range z.serverPools {
-		deletedObjects, errs := zone.DeleteObjects(ctx, bucket, objects, opts)
+	for _, pool := range z.serverPools {
+		deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objects, opts)
 		for i, derr := range errs {
 			if derr != nil {
 				derrs[i] = derr
@@ -620,7 +633,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
-	zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
+	poolIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
@@ -628,12 +641,12 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 	if cpSrcDstSame && srcInfo.metadataOnly {
 		// Version ID is set for the destination and source == destination version ID.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
-			return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+			return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		// Destination is not versioned and source version ID is empty
 		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
-			return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+			return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		// Destination is versioned, source is not destination version,
 		// as a special case look for if the source object is not legacy
@@ -643,7 +656,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 			// CopyObject optimization where we don't create an entire copy
 			// of the content, instead we add a reference.
 			srcInfo.versionOnly = true
-			return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+			return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 	}
 
@@ -655,7 +668,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		MTime:                dstOpts.MTime,
 	}
 
-	return z.serverPools[zoneIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
+	return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
 }
 
 func (z *erasureServerPools) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
@@ -701,7 +714,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	// more objects matching the prefix.
 	ri := logger.GetReqInfo(ctx)
 	if ri != nil && strings.Contains(ri.UserAgent, `1.0 Veeam/1.0 Backup`) && strings.HasSuffix(prefix, ".blk") {
-		opts.singleObject = true
+		opts.discardResult = true
 		opts.Transient = true
 	}
 
@@ -709,14 +722,21 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	if err != nil && err != io.EOF {
 		return loi, err
 	}
-	loi.Objects, loi.Prefixes = merged.fileInfoVersions(bucket, prefix, delimiter, versionMarker)
-	loi.IsTruncated = err == nil && len(loi.Objects) > 0
-	if maxKeys > 0 && len(loi.Objects) > maxKeys {
-		loi.Objects = loi.Objects[:maxKeys]
+	objects := merged.fileInfoVersions(bucket, prefix, delimiter, versionMarker)
+	loi.IsTruncated = err == nil && len(objects) > 0
+	if maxKeys > 0 && len(objects) > maxKeys {
+		objects = objects[:maxKeys]
 		loi.IsTruncated = true
 	}
+	for _, obj := range objects {
+		if obj.IsDir && delimiter != "" {
+			loi.Prefixes = append(loi.Prefixes, obj.Name)
+		} else {
+			loi.Objects = append(loi.Objects, obj)
+		}
+	}
 	if loi.IsTruncated {
-		last := loi.Objects[len(loi.Objects)-1]
+		last := objects[len(objects)-1]
 		loi.NextMarker = encodeMarker(last.Name, merged.listID)
 		loi.NextVersionIDMarker = last.VersionID
 	}
@@ -725,6 +745,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 
 func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
+
 	merged, err := z.listPath(ctx, listPathOptions{
 		Bucket:      bucket,
 		Prefix:      prefix,
@@ -738,11 +759,24 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		logger.LogIf(ctx, err)
 		return loi, err
 	}
+
 	// Default is recursive, if delimiter is set then list non recursive.
-	loi.Objects, loi.Prefixes = merged.fileInfos(bucket, prefix, delimiter)
-	loi.IsTruncated = err == nil && len(loi.Objects) > 0
+	objects := merged.fileInfos(bucket, prefix, delimiter)
+	loi.IsTruncated = err == nil && len(objects) > 0
+	if maxKeys > 0 && len(objects) > maxKeys {
+		objects = objects[:maxKeys]
+		loi.IsTruncated = true
+	}
+	for _, obj := range objects {
+		if obj.IsDir && delimiter != "" {
+			loi.Prefixes = append(loi.Prefixes, obj.Name)
+		} else {
+			loi.Objects = append(loi.Objects, obj)
+		}
+	}
 	if loi.IsTruncated {
-		loi.NextMarker = encodeMarker(loi.Objects[len(loi.Objects)-1].Name, merged.listID)
+		last := objects[len(objects)-1]
+		loi.NextMarker = encodeMarker(last.Name, merged.listID)
 	}
 	return loi, nil
 }
@@ -756,20 +790,20 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 		return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	}
 
-	var zoneResult = ListMultipartsInfo{}
-	zoneResult.MaxUploads = maxUploads
-	zoneResult.KeyMarker = keyMarker
-	zoneResult.Prefix = prefix
-	zoneResult.Delimiter = delimiter
-	for _, zone := range z.serverPools {
-		result, err := zone.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
+	var poolResult = ListMultipartsInfo{}
+	poolResult.MaxUploads = maxUploads
+	poolResult.KeyMarker = keyMarker
+	poolResult.Prefix = prefix
+	poolResult.Delimiter = delimiter
+	for _, pool := range z.serverPools {
+		result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
 			delimiter, maxUploads)
 		if err != nil {
 			return result, err
 		}
-		zoneResult.Uploads = append(zoneResult.Uploads, result.Uploads...)
+		poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
 	}
-	return zoneResult, nil
+	return poolResult, nil
 }
 
 // Initiate a new multipart upload on a hashedSet based on object name.
@@ -811,14 +845,14 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
-	for _, zone := range z.serverPools {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
+			return pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
-			// Look for information on the next zone
+			// Look for information on the next pool
 			continue
 		}
 		// Any other unhandled errors such as quorum return.
@@ -840,14 +874,14 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 	if z.SingleZone() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
-	for _, zone := range z.serverPools {
-		mi, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		mi, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return mi, nil
 		}
 		switch err.(type) {
 		case InvalidUploadID:
-			// upload id not found, continue to the next zone.
+			// upload id not found, continue to the next pool.
 			continue
 		}
 		// any other unhandled error return right here.
@@ -870,10 +904,10 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 	if z.SingleZone() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
-	for _, zone := range z.serverPools {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
+			return pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -898,14 +932,14 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
-	for _, zone := range z.serverPools {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
+			return pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
-			// upload id not found move to next zone
+			// upload id not found move to next pool
 			continue
 		}
 		return err
@@ -928,17 +962,17 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 	}
 
 	// Purge any existing object.
-	for _, zone := range z.serverPools {
-		zone.DeleteObject(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		pool.DeleteObject(ctx, bucket, object, opts)
 	}
 
-	for _, zone := range z.serverPools {
-		result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
+	for _, pool := range z.serverPools {
+		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
 		if err != nil {
 			return objInfo, err
 		}
 		if result.Lookup(uploadID) {
-			return zone.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
+			return pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 		}
 	}
 	return objInfo, InvalidUploadID{
@@ -961,8 +995,8 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (
 		}
 		return bucketInfo, nil
 	}
-	for _, zone := range z.serverPools {
-		bucketInfo, err = zone.GetBucketInfo(ctx, bucket)
+	for _, pool := range z.serverPools {
+		bucketInfo, err = pool.GetBucketInfo(ctx, bucket)
 		if err != nil {
 			if isErrBucketNotFound(err) {
 				continue
@@ -1087,8 +1121,8 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
 	if z.SingleZone() {
 		buckets, err = z.serverPools[0].ListBuckets(ctx)
 	} else {
-		for _, zone := range z.serverPools {
-			buckets, err = zone.ListBuckets(ctx)
+		for _, pool := range z.serverPools {
+			buckets, err = pool.ListBuckets(ctx)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
@@ -1122,8 +1156,8 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 	}
 
 	var countNoHeal int
-	for _, zone := range z.serverPools {
-		result, err := zone.HealFormat(ctx, dryRun)
+	for _, pool := range z.serverPools {
+		result, err := pool.HealFormat(ctx, dryRun)
 		if err != nil && !errors.Is(err, errNoHealRequired) {
 			logger.LogIf(ctx, err)
 			continue
@@ -1147,14 +1181,17 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 	return r, nil
 }
 
-func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error) {
+func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	var r = madmin.HealResultItem{
 		Type:   madmin.HealItemBucket,
 		Bucket: bucket,
 	}
 
-	for _, zone := range z.serverPools {
-		result, err := zone.HealBucket(ctx, bucket, dryRun, remove)
+	// Attempt heal on the bucket metadata, ignore any failures
+	_, _ = z.HealObject(ctx, minioMetaBucket, pathJoin(bucketConfigPrefix, bucket, bucketMetadataFile), "", opts)
+
+	for _, pool := range z.serverPools {
+		result, err := pool.HealBucket(ctx, bucket, opts)
 		if err != nil {
 			switch err.(type) {
 			case BucketNotFound:
@@ -1167,6 +1204,7 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, dryR
 		r.Before.Drives = append(r.Before.Drives, result.Before.Drives...)
 		r.After.Drives = append(r.After.Drives, result.After.Drives...)
 	}
+
 	return r, nil
 }
 
@@ -1313,8 +1351,8 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 		defer lk.RUnlock()
 	}
 
-	for _, zone := range z.serverPools {
-		result, err := zone.HealObject(ctx, bucket, object, versionID, opts)
+	for _, pool := range z.serverPools {
+		result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1336,26 +1374,6 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 	}
 }
 
-func (z *erasureServerPools) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
-	var healBuckets []BucketInfo
-	for _, zone := range z.serverPools {
-		bucketsInfo, err := zone.ListBucketsHeal(ctx)
-		if err != nil {
-			continue
-		}
-		healBuckets = append(healBuckets, bucketsInfo...)
-	}
-
-	for i := range healBuckets {
-		meta, err := globalBucketMetadataSys.Get(healBuckets[i].Name)
-		if err == nil {
-			healBuckets[i].Created = meta.Created
-		}
-	}
-
-	return healBuckets, nil
-}
-
 // GetMetrics - no op
 func (z *erasureServerPools) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
@@ -1363,12 +1381,12 @@ func (z *erasureServerPools) GetMetrics(ctx context.Context) (*Metrics, error) {
 }
 
 func (z *erasureServerPools) getZoneAndSet(id string) (int, int, error) {
-	for zoneIdx := range z.serverPools {
-		format := z.serverPools[zoneIdx].format
+	for poolIdx := range z.serverPools {
+		format := z.serverPools[poolIdx].format
 		for setIdx, set := range format.Erasure.Sets {
 			for _, diskID := range set {
 				if diskID == id {
-					return zoneIdx, setIdx, nil
+					return poolIdx, setIdx, nil
 				}
 			}
 		}
@@ -1408,12 +1426,12 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 	for _, localDiskIDs := range diskIDs {
 		for _, id := range localDiskIDs {
-			zoneIdx, setIdx, err := z.getZoneAndSet(id)
+			poolIdx, setIdx, err := z.getZoneAndSet(id)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
 			}
-			erasureSetUpCount[zoneIdx][setIdx]++
+			erasureSetUpCount[poolIdx][setIdx]++
 		}
 	}
 
@@ -1450,16 +1468,16 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		}
 	}
 
-	for zoneIdx := range erasureSetUpCount {
-		for setIdx := range erasureSetUpCount[zoneIdx] {
-			if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
+	for poolIdx := range erasureSetUpCount {
+		for setIdx := range erasureSetUpCount[poolIdx] {
+			if erasureSetUpCount[poolIdx][setIdx] < writeQuorum {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
-					fmt.Errorf("Write quorum may be lost on zone: %d, set: %d, expected write quorum: %d",
-						zoneIdx, setIdx, writeQuorum))
+					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
+						poolIdx, setIdx, writeQuorum))
 				return HealthResult{
 					Healthy:       false,
 					HealingDrives: len(aggHealStateResult.HealDisks),
-					ZoneID:        zoneIdx,
+					ZoneID:        poolIdx,
 					SetID:         setIdx,
 					WriteQuorum:   writeQuorum,
 				}
@@ -1490,8 +1508,8 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
 
-	for _, zone := range z.serverPools {
-		err := zone.PutObjectTags(ctx, bucket, object, tags, opts)
+	for _, pool := range z.serverPools {
+		err := pool.PutObjectTags(ctx, bucket, object, tags, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1519,8 +1537,8 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 	if z.SingleZone() {
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
-	for _, zone := range z.serverPools {
-		err := zone.DeleteObjectTags(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		err := pool.DeleteObjectTags(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1548,8 +1566,8 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 	if z.SingleZone() {
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
-	for _, zone := range z.serverPools {
-		tags, err := zone.GetObjectTags(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		tags, err := pool.GetObjectTags(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue

cmd/erasure-sets.go

@@ -30,9 +30,11 @@ import (
 
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -244,10 +246,13 @@ func (s *erasureSets) connectDisks() {
 			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			s.erasureDisksMu.Unlock()
 			go func(setIndex int) {
+				idler := time.NewTimer(100 * time.Millisecond)
+				defer idler.Stop()
+
 				// Send a new disk connect event with a timeout
 				select {
 				case s.disksConnectEvent <- diskConnectInfo{setIndex: setIndex}:
-				case <-time.After(100 * time.Millisecond):
+				case <-idler.C:
 				}
 			}(setIndex)
 		}(endpoint)
@@ -266,11 +271,21 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	// Pre-emptively connect the disks if possible.
 	s.connectDisks()
 
+	monitor := time.NewTimer(monitorInterval)
+	defer monitor.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-time.After(monitorInterval):
+		case <-monitor.C:
+			// Reset the timer once fired for required interval.
+			monitor.Reset(monitorInterval)
+
+			if serverDebugLog {
+				console.Debugln("running disk monitoring")
+			}
+
 			s.connectDisks()
 		}
 	}
@@ -278,10 +293,20 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 
 // GetAllLockers return a list of all lockers for all sets.
 func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
-	allLockers := make([]dsync.NetLocker, s.setDriveCount*s.setCount)
-	for i, lockers := range s.erasureLockers {
-		for j, locker := range lockers {
-			allLockers[i*s.setDriveCount+j] = locker
+	var allLockers []dsync.NetLocker
+	lockEpSet := set.NewStringSet()
+	for _, lockers := range s.erasureLockers {
+		for _, locker := range lockers {
+			if locker == nil || !locker.IsOnline() {
+				// Skip any offline lockers.
+				continue
+			}
+			if lockEpSet.Contains(locker.String()) {
+				// Skip duplicate lockers.
+				continue
+			}
+			lockEpSet.Add(locker.String())
+			allLockers = append(allLockers, locker)
 		}
 	}
 	return allLockers
@@ -289,15 +314,8 @@ func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
 
 func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) {
 	return func() ([]dsync.NetLocker, string) {
-		lockers := make([]dsync.NetLocker, s.setDriveCount)
+		lockers := make([]dsync.NetLocker, len(s.erasureLockers[setIndex]))
 		copy(lockers, s.erasureLockers[setIndex])
-		sort.Slice(lockers, func(i, j int) bool {
-			// re-order lockers with affinity for
-			// - non-local lockers
-			// - online lockers
-			// are used first
-			return !lockers[i].IsLocal() && lockers[i].IsOnline()
-		})
 		return lockers, s.erasureLockOwner
 	}
 }
@@ -362,14 +380,24 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 	for i := 0; i < setCount; i++ {
 		s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
-		s.erasureLockers[i] = make([]dsync.NetLocker, setDriveCount)
+	}
+
+	var erasureLockers = map[string]dsync.NetLocker{}
+	for _, endpoint := range endpoints {
+		if _, ok := erasureLockers[endpoint.Host]; !ok {
+			erasureLockers[endpoint.Host] = newLockAPI(endpoint)
+		}
 	}
 
 	for i := 0; i < setCount; i++ {
+		var lockerEpSet = set.NewStringSet()
 		for j := 0; j < setDriveCount; j++ {
 			endpoint := endpoints[i*setDriveCount+j]
-			// Rely on endpoints list to initialize, init lockers and available disks.
-			s.erasureLockers[i][j] = newLockAPI(endpoint)
+			// Only add lockers only one per endpoint and per erasure set.
+			if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) {
+				lockerEpSet.Add(endpoint.Host)
+				s.erasureLockers[i] = append(s.erasureLockers[i], locker)
+			}
 			disk := storageDisks[i*setDriveCount+j]
 			if disk == nil {
 				continue
@@ -388,18 +416,19 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 		// Initialize erasure objects for a given set.
 		s.sets[i] = &erasureObjects{
-			getDisks:     s.GetDisks(i),
-			getLockers:   s.GetLockers(i),
-			getEndpoints: s.GetEndpoints(i),
-			nsMutex:      mutex,
-			bp:           bp,
-			mrfOpCh:      make(chan partialOperation, 10000),
+			setDriveCount: setDriveCount,
+			getDisks:      s.GetDisks(i),
+			getLockers:    s.GetLockers(i),
+			getEndpoints:  s.GetEndpoints(i),
+			nsMutex:       mutex,
+			bp:            bp,
+			mrfOpCh:       make(chan partialOperation, 10000),
 		}
-
-		go s.sets[i].cleanupStaleUploads(ctx,
-			GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
 	}
 
+	// start cleanup stale uploads go-routine.
+	go s.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
+
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
 	go s.maintainMRFList()
@@ -408,6 +437,22 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	return s, nil
 }
 
+func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
+	ticker := time.NewTicker(cleanupInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			for _, set := range s.sets {
+				set.cleanupStaleUploads(ctx, expiry)
+			}
+		}
+	}
+}
+
 // NewNSLock - initialize a new namespace RWLocker instance.
 func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
 	if len(objects) == 1 {
@@ -436,7 +481,7 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
 			index := index
 			g.Go(func() error {
 				// ignoring errors on purpose
-				storageInfos[index], _ = s.sets[index].StorageInfo(ctx, false)
+				storageInfos[index], _ = s.sets[index].StorageInfo(ctx)
 				return nil
 			}, index)
 		}
@@ -446,8 +491,6 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
 
 		for _, lstorageInfo := range storageInfos {
 			storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
-			storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
-			storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
 		}
 
 		return storageInfo
@@ -465,7 +508,7 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
 }
 
 // StorageInfo - combines output of StorageInfo across all erasure coded object sets.
-func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
+func (s *erasureSets) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 	var storageInfo StorageInfo
 
 	storageInfos := make([]StorageInfo, len(s.sets))
@@ -475,7 +518,7 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 	for index := range s.sets {
 		index := index
 		g.Go(func() error {
-			storageInfos[index], storageInfoErrs[index] = s.sets[index].StorageInfo(ctx, local)
+			storageInfos[index], storageInfoErrs[index] = s.sets[index].StorageInfo(ctx)
 			return nil
 		}, index)
 	}
@@ -485,14 +528,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 
 	for _, lstorageInfo := range storageInfos {
 		storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
-		storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
-		storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
-	}
-
-	if local {
-		// if local is true, we are not interested in the drive UUID info.
-		// this is called primarily by prometheus
-		return storageInfo, nil
 	}
 
 	var errs []error
@@ -683,8 +718,24 @@ func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObj
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all sets.
 func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
-	// Always lists from the same set signified by the empty string.
-	return s.ListBucketsHeal(ctx)
+	var listBuckets []BucketInfo
+	var healBuckets = map[string]VolInfo{}
+	for _, set := range s.sets {
+		// lists all unique buckets across drives.
+		if err := listAllBuckets(ctx, set.getDisks(), healBuckets); err != nil {
+			return nil, err
+		}
+	}
+
+	for _, v := range healBuckets {
+		listBuckets = append(listBuckets, BucketInfo(v))
+	}
+
+	sort.Slice(listBuckets, func(i, j int) bool {
+		return listBuckets[i].Name < listBuckets[j].Name
+	})
+
+	return listBuckets, nil
 }
 
 // --- Object Operations ---
@@ -1230,7 +1281,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 }
 
 // HealBucket - heals inconsistent buckets and bucket metadata on all sets.
-func (s *erasureSets) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (result madmin.HealResultItem, err error) {
+func (s *erasureSets) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	// Initialize heal result info
 	result = madmin.HealResultItem{
 		Type:      madmin.HealItemBucket,
@@ -1241,7 +1292,7 @@ func (s *erasureSets) HealBucket(ctx context.Context, bucket string, dryRun, rem
 
 	for _, s := range s.sets {
 		var healResult madmin.HealResultItem
-		healResult, err = s.HealBucket(ctx, bucket, dryRun, remove)
+		healResult, err = s.HealBucket(ctx, bucket, opts)
 		if err != nil {
 			return result, err
 		}
@@ -1263,23 +1314,6 @@ func (s *erasureSets) HealObject(ctx context.Context, bucket, object, versionID
 	return s.getHashedSet(object).HealObject(ctx, bucket, object, versionID, opts)
 }
 
-// Lists all buckets which need healing.
-func (s *erasureSets) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
-	var listBuckets []BucketInfo
-	var healBuckets = map[string]VolInfo{}
-	for _, set := range s.sets {
-		// lists all unique buckets across drives.
-		if err := listAllBuckets(ctx, set.getDisks(), healBuckets); err != nil {
-			return nil, err
-		}
-	}
-	for _, v := range healBuckets {
-		listBuckets = append(listBuckets, BucketInfo(v))
-	}
-	sort.Sort(byBucketName(listBuckets))
-	return listBuckets, nil
-}
-
 // PutObjectTags - replace or add tags to an existing object
 func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) error {
 	return s.getHashedSet(object).PutObjectTags(ctx, bucket, object, tags, opts)
@@ -1327,23 +1361,34 @@ func (s *erasureSets) maintainMRFList() {
 	}
 }
 
+func toSourceChTimed(t *time.Timer, sourceCh chan healSource, u healSource) {
+	t.Reset(100 * time.Millisecond)
+
+	// No defer, as we don't know which
+	// case will be selected
+
+	select {
+	case sourceCh <- u:
+	case <-t.C:
+		return
+	}
+
+	// We still need to check the return value
+	// of Stop, because t could have fired
+	// between the send on sourceCh and this line.
+	if !t.Stop() {
+		<-t.C
+	}
+}
+
 // healMRFRoutine monitors new disks connection, sweep the MRF list
 // to find objects related to the new disk that needs to be healed.
 func (s *erasureSets) healMRFRoutine() {
 	// Wait until background heal state is initialized
-	var bgSeq *healSequence
-	for {
-		if globalBackgroundHealState == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-		var ok bool
-		bgSeq, ok = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-		if ok {
-			break
-		}
-		time.Sleep(time.Second)
-	}
+	bgSeq := mustGetHealSequence(GlobalContext)
+
+	idler := time.NewTimer(100 * time.Millisecond)
+	defer idler.Stop()
 
 	for e := range s.disksConnectEvent {
 		// Get the list of objects related the er.set
@@ -1359,11 +1404,8 @@ func (s *erasureSets) healMRFRoutine() {
 
 		// Heal objects
 		for _, u := range mrfOperations {
-			// Send an object to be healed with a timeout
-			select {
-			case bgSeq.sourceCh <- u:
-			case <-time.After(100 * time.Millisecond):
-			}
+			// Send an object to background heal
+			toSourceChTimed(idler, bgSeq.sourceCh, u)
 
 			s.mrfMU.Lock()
 			delete(s.mrfOperations, u)

cmd/erasure-utils.go

@@ -77,6 +77,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 			// from subsequent blocks.
 			offset = 0
 		}
+
 		// We have written all the blocks, write the last remaining block.
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
@@ -89,6 +90,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 			totalWritten += n
 			break
 		}
+
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {